From 13e848ddf4cfa8f4cbfb672c1cb8ffb4f304646e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 8 Jun 2026 16:52:41 +0800 Subject: [PATCH 01/33] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=8A=80?= =?UTF-8?q?=E8=83=BD=E7=AE=A1=E7=90=86=E6=A8=A1=E5=9D=97=E8=AF=95=E9=AA=8C?= =?UTF-8?q?=E6=80=A7=E5=8A=9F=E8=83=BD=EF=BC=8C=E7=AD=89=E5=BE=85=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- easy-agents-bom/pom.xml | 4 + easy-agents-skill/pom.xml | 27 ++ .../skill/codec/SkillPackageCodec.java | 20 ++ .../skill/codec/ZipSkillPackageCodec.java | 254 ++++++++++++++++ .../skill/exception/SkillException.java | 26 ++ .../exception/SkillPackageException.java | 26 ++ .../exception/SkillValidationException.java | 26 ++ .../skill/factory/SkillFactory.java | 52 ++++ .../com/easyagents/skill/model/Skill.java | 175 +++++++++++ .../easyagents/skill/model/SkillAsset.java | 145 +++++++++ .../skill/model/SkillDescriptor.java | 111 +++++++ .../easyagents/skill/model/SkillMetadata.java | 79 +++++ .../skill/model/SkillReference.java | 126 ++++++++ .../easyagents/skill/model/SkillScript.java | 126 ++++++++ .../skill/model/SkillScriptLanguage.java | 50 ++++ .../skill/repository/SkillRepository.java | 58 ++++ .../memory/InMemorySkillRepository.java | 176 +++++++++++ .../skill/store/SkillContentStore.java | 41 +++ .../memory/InMemorySkillContentStore.java | 70 +++++ .../skill/util/SkillFrontmatter.java | 89 ++++++ .../easyagents/skill/util/SkillHashes.java | 47 +++ .../com/easyagents/skill/util/SkillPaths.java | 118 ++++++++ .../skill/validation/SkillValidator.java | 16 + .../defaults/DefaultSkillValidator.java | 156 ++++++++++ .../skill/codec/ZipSkillPackageCodecTest.java | 275 ++++++++++++++++++ .../memory/InMemorySkillRepositoryTest.java | 83 ++++++ .../memory/InMemorySkillContentStoreTest.java | 46 +++ .../defaults/DefaultSkillValidatorTest.java | 193 ++++++++++++ pom.xml | 7 + 29 files changed, 2622 insertions(+) create mode 100644 easy-agents-skill/pom.xml create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillException.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java diff --git a/easy-agents-bom/pom.xml b/easy-agents-bom/pom.xml index 4911bcc..32234e4 100644 --- a/easy-agents-bom/pom.xml +++ b/easy-agents-bom/pom.xml @@ -256,6 +256,10 @@ + + com.easyagents + easy-agents-skill + com.easyagents easy-agents-agent-runtime diff --git a/easy-agents-skill/pom.xml b/easy-agents-skill/pom.xml new file mode 100644 index 0000000..8dbe3b6 --- /dev/null +++ b/easy-agents-skill/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + com.easyagents + easy-agents + ${revision} + + + easy-agents-skill + easy-agents-skill + + + 17 + UTF-8 + + + + + junit + junit + test + + + diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java new file mode 100644 index 0000000..7936b0e --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java @@ -0,0 +1,20 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.model.Skill; + +import java.io.InputStream; +import java.util.List; + +/** + * Skill 包导入接口。 + */ +public interface SkillPackageCodec { + + /** + * 从 zip 输入流导入 Skill。 + * + * @param inputStream zip 输入流 + * @return Skill 列表 + */ + List importZip(InputStream inputStream); +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java new file mode 100644 index 0000000..16590af --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java @@ -0,0 +1,254 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.exception.SkillPackageException; +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.factory.SkillFactory; +import com.easyagents.skill.model.*; +import com.easyagents.skill.store.SkillContentStore; +import com.easyagents.skill.store.memory.InMemorySkillContentStore; +import com.easyagents.skill.util.SkillHashes; +import com.easyagents.skill.util.SkillPaths; +import com.easyagents.skill.validation.SkillValidator; +import com.easyagents.skill.validation.defaults.DefaultSkillValidator; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * 基于 zip 的 Skill 包导入实现。 + */ +public class ZipSkillPackageCodec implements SkillPackageCodec { + + private static final String DEFAULT_MEDIA_TYPE = "application/octet-stream"; + + private final SkillContentStore contentStore; + private final SkillValidator validator; + + /** + * 创建使用内存内容存储的 zip Skill 包导入器。 + */ + public ZipSkillPackageCodec() { + this(new InMemorySkillContentStore()); + } + + /** + * 创建 zip Skill 包导入器。 + * + * @param contentStore 二进制内容存储 + */ + public ZipSkillPackageCodec(SkillContentStore contentStore) { + this(contentStore, new DefaultSkillValidator()); + } + + /** + * 创建 zip Skill 包导入器。 + * + * @param contentStore 二进制内容存储 + * @param validator Skill 聚合校验器 + */ + public ZipSkillPackageCodec(SkillContentStore contentStore, SkillValidator validator) { + if (contentStore == null) { + throw new SkillPackageException("Skill content store is required."); + } + if (validator == null) { + throw new SkillPackageException("Skill validator is required."); + } + this.contentStore = contentStore; + this.validator = validator; + } + + /** + * 从 zip 流导入 Skill 列表。 + * + * @param inputStream zip 输入流 + * @return Skill 列表 + */ + @Override + public List importZip(InputStream inputStream) { + if (inputStream == null) { + throw new SkillPackageException("Zip input stream is required."); + } + Map builders = new LinkedHashMap<>(); + try (ZipInputStream zipInputStream = new ZipInputStream(inputStream)) { + ZipEntry entry; + while ((entry = zipInputStream.getNextEntry()) != null) { + handleEntry(entry, zipInputStream, builders); + zipInputStream.closeEntry(); + } + } catch (IOException e) { + throw new SkillPackageException("Failed to import skill zip package.", e); + } + if (builders.isEmpty()) { + throw new SkillPackageException("Zip package must contain at least one skill folder."); + } + List skills = new ArrayList<>(); + for (SkillBuilder builder : builders.values()) { + Skill skill = builder.build(); + validator.validate(skill); + skills.add(skill); + } + for (SkillBuilder builder : builders.values()) { + builder.writeAssets(contentStore); + } + return skills; + } + + private void handleEntry(ZipEntry entry, ZipInputStream zipInputStream, Map builders) + throws IOException { + String rawPath = entry.getName(); + if (entry.isDirectory() || SkillPaths.isIgnoredSystemPath(rawPath)) { + return; + } + String normalizedPath = SkillPaths.normalize(rawPath); + if (SkillPaths.SKILL_FILE.equals(normalizedPath)) { + throw new SkillPackageException("Zip root cannot directly contain SKILL.md."); + } + int slashIndex = normalizedPath.indexOf('/'); + if (slashIndex < 0) { + throw new SkillPackageException("Zip root can only contain skill folders: " + normalizedPath); + } + + String skillId = normalizedPath.substring(0, slashIndex); + String skillPath = normalizedPath.substring(slashIndex + 1); + if (skillPath.isBlank()) { + return; + } + SkillBuilder builder = builders.computeIfAbsent(skillId, SkillBuilder::new); + builder.addFile(skillPath, readEntryBytes(zipInputStream)); + } + + private static byte[] readEntryBytes(InputStream inputStream) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int readLength; + while ((readLength = inputStream.read(buffer)) >= 0) { + outputStream.write(buffer, 0, readLength); + } + return outputStream.toByteArray(); + } + + private static String detectMediaType(String path) { + String mediaType = URLConnection.guessContentTypeFromName(path); + return mediaType == null ? DEFAULT_MEDIA_TYPE : mediaType; + } + + private static final class SkillBuilder { + + private final String id; + private final Set paths = new HashSet<>(); + private String skillContent; + private final List references = new ArrayList<>(); + private final List scripts = new ArrayList<>(); + private final List assets = new ArrayList<>(); + private final List pendingAssets = new ArrayList<>(); + + private SkillBuilder(String id) { + this.id = id; + } + + private void addFile(String path, byte[] bytes) { + String normalizedPath = SkillPaths.normalize(path); + if (!paths.add(normalizedPath)) { + throw new SkillPackageException("Duplicate skill file path: " + id + "/" + normalizedPath); + } + if (SkillPaths.SKILL_FILE.equals(normalizedPath)) { + addSkillFile(bytes); + return; + } + + String topDir = SkillPaths.firstSegment(normalizedPath); + if (SkillPaths.REFERENCES_DIR.equals(topDir)) { + addReference(normalizedPath, bytes); + } else if (SkillPaths.SCRIPTS_DIR.equals(topDir)) { + addScript(normalizedPath, bytes); + } else if (SkillPaths.ASSETS_DIR.equals(topDir)) { + addAsset(normalizedPath, bytes); + } else { + throw new SkillPackageException("Unknown skill top-level directory: " + id + "/" + normalizedPath); + } + } + + private void addSkillFile(byte[] bytes) { + skillContent = new String(bytes, StandardCharsets.UTF_8); + } + + private void addReference(String path, byte[] bytes) { + if (!SkillPaths.hasExtension(path, ".md")) { + throw new SkillPackageException("Skill reference must be a markdown file: " + id + "/" + path); + } + String content = new String(bytes, StandardCharsets.UTF_8); + SkillReference reference = new SkillReference(); + reference.setPath(path); + reference.setName(SkillPaths.fileName(path)); + reference.setContent(content); + reference.setContentHash(SkillHashes.sha256Hex(bytes)); + reference.setSize(bytes.length); + references.add(reference); + } + + private void addScript(String path, byte[] bytes) { + SkillScriptLanguage language = SkillScriptLanguage.fromPath(path); + if (language == SkillScriptLanguage.UNKNOWN) { + throw new SkillPackageException("Unsupported skill script extension: " + id + "/" + path); + } + SkillScript script = new SkillScript(); + script.setPath(path); + script.setLanguage(language); + script.setContent(new String(bytes, StandardCharsets.UTF_8)); + script.setContentHash(SkillHashes.sha256Hex(bytes)); + script.setSize(bytes.length); + scripts.add(script); + } + + private void addAsset(String path, byte[] bytes) { + String contentHash = SkillHashes.sha256Hex(bytes); + SkillAsset asset = new SkillAsset(); + asset.setPath(path); + asset.setName(SkillPaths.fileName(path)); + asset.setMediaType(detectMediaType(path)); + asset.setContentRef("sha256:" + contentHash); + asset.setContentHash(contentHash); + asset.setSize(bytes.length); + assets.add(asset); + pendingAssets.add(new PendingAsset(asset, bytes)); + } + + private Skill build() { + if (skillContent == null) { + throw new SkillPackageException("Skill folder must contain SKILL.md: " + id); + } + try { + return SkillFactory.create(id, skillContent, references, scripts, assets); + } catch (SkillValidationException e) { + throw new SkillPackageException("Invalid SKILL.md frontmatter: " + id, e); + } + } + + private void writeAssets(SkillContentStore contentStore) { + for (PendingAsset pendingAsset : pendingAssets) { + String contentRef = contentStore.put(pendingAsset.bytes); + if (!pendingAsset.asset.getContentRef().equals(contentRef)) { + throw new SkillPackageException("Skill asset content ref does not match store result: " + + id + "/" + pendingAsset.asset.getPath()); + } + } + } + } + + private static final class PendingAsset { + + private final SkillAsset asset; + private final byte[] bytes; + + private PendingAsset(SkillAsset asset, byte[] bytes) { + this.asset = asset; + this.bytes = bytes; + } + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillException.java b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillException.java new file mode 100644 index 0000000..b263760 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillException.java @@ -0,0 +1,26 @@ +package com.easyagents.skill.exception; + +/** + * Skill 模块基础运行时异常。 + */ +public class SkillException extends RuntimeException { + + /** + * 创建 Skill 异常。 + * + * @param message 异常信息 + */ + public SkillException(String message) { + super(message); + } + + /** + * 创建 Skill 异常。 + * + * @param message 异常信息 + * @param cause 原始异常 + */ + public SkillException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java new file mode 100644 index 0000000..ef388ae --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java @@ -0,0 +1,26 @@ +package com.easyagents.skill.exception; + +/** + * Skill 包导入导出异常。 + */ +public class SkillPackageException extends SkillException { + + /** + * 创建 Skill 包异常。 + * + * @param message 异常信息 + */ + public SkillPackageException(String message) { + super(message); + } + + /** + * 创建 Skill 包异常。 + * + * @param message 异常信息 + * @param cause 原始异常 + */ + public SkillPackageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java new file mode 100644 index 0000000..dbf781c --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java @@ -0,0 +1,26 @@ +package com.easyagents.skill.exception; + +/** + * Skill 校验失败异常。 + */ +public class SkillValidationException extends SkillException { + + /** + * 创建 Skill 校验异常。 + * + * @param message 异常信息 + */ + public SkillValidationException(String message) { + super(message); + } + + /** + * 创建 Skill 校验异常。 + * + * @param message 异常信息 + * @param cause 原始异常 + */ + public SkillValidationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java b/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java new file mode 100644 index 0000000..2a899cd --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java @@ -0,0 +1,52 @@ +package com.easyagents.skill.factory; + +import com.easyagents.skill.model.*; +import com.easyagents.skill.util.SkillFrontmatter; + +import java.util.List; +import java.util.Map; + +/** + * Skill 标准构建入口。 + */ +public final class SkillFactory { + + private SkillFactory() { + } + + /** + * 基于 SKILL.md 内容创建 Skill。 + * + * @param id Skill ID + * @param skillContent SKILL.md 原始内容 + * @return Skill 聚合 + */ + public static Skill create(String id, String skillContent) { + return create(id, skillContent, null, null, null); + } + + /** + * 基于 SKILL.md 内容和资源列表创建 Skill。 + * + * @param id Skill ID + * @param skillContent SKILL.md 原始内容 + * @param references reference 文档列表 + * @param scripts script 脚本列表 + * @param assets asset 资产列表 + * @return Skill 聚合 + */ + public static Skill create(String id, String skillContent, List references, + List scripts, List assets) { + Map values = SkillFrontmatter.parse(skillContent); + Skill skill = new Skill(); + skill.setId(id); + skill.setName(values.get("name").toString()); + skill.setDescription(values.get("description").toString()); + skill.setMetadata(new SkillMetadata(values)); + skill.setSkillContent(skillContent); + skill.setReferences(references); + skill.setScripts(scripts); + skill.setAssets(assets); + return skill; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java new file mode 100644 index 0000000..e845538 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java @@ -0,0 +1,175 @@ +package com.easyagents.skill.model; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Skill 聚合根。 + */ +public class Skill implements Serializable { + + private static final long serialVersionUID = 1L; + + private String id; + private String name; + private String description; + private SkillMetadata metadata = new SkillMetadata(); + private String skillContent; + private List references = new ArrayList<>(); + private List scripts = new ArrayList<>(); + private List assets = new ArrayList<>(); + + /** + * 获取 Skill ID。 + * + * @return Skill ID + */ + public String getId() { + return id; + } + + /** + * 设置 Skill ID。 + * + * @param id Skill ID + */ + public void setId(String id) { + this.id = id; + } + + /** + * 获取名称。 + * + * @return 名称 + */ + public String getName() { + return name; + } + + /** + * 设置名称。 + * + * @param name 名称 + */ + public void setName(String name) { + this.name = name; + } + + /** + * 获取描述。 + * + * @return 描述 + */ + public String getDescription() { + return description; + } + + /** + * 设置描述。 + * + * @param description 描述 + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * 获取元数据。 + * + * @return 元数据 + */ + public SkillMetadata getMetadata() { + return metadata; + } + + /** + * 设置元数据。 + * + * @param metadata 元数据 + */ + public void setMetadata(SkillMetadata metadata) { + this.metadata = metadata == null ? new SkillMetadata() : metadata; + } + + /** + * 获取 SKILL.md 原始内容。 + * + * @return SKILL.md 原始内容 + */ + public String getSkillContent() { + return skillContent; + } + + /** + * 设置 SKILL.md 原始内容。 + * + * @param skillContent SKILL.md 原始内容 + */ + public void setSkillContent(String skillContent) { + this.skillContent = skillContent; + } + + /** + * 获取参考文档列表。 + * + * @return 参考文档列表 + */ + public List getReferences() { + return references; + } + + /** + * 设置参考文档列表。 + * + * @param references 参考文档列表 + */ + public void setReferences(List references) { + this.references = references == null ? new ArrayList<>() : new ArrayList<>(references); + } + + /** + * 获取脚本列表。 + * + * @return 脚本列表 + */ + public List getScripts() { + return scripts; + } + + /** + * 设置脚本列表。 + * + * @param scripts 脚本列表 + */ + public void setScripts(List scripts) { + this.scripts = scripts == null ? new ArrayList<>() : new ArrayList<>(scripts); + } + + /** + * 获取资产列表。 + * + * @return 资产列表 + */ + public List getAssets() { + return assets; + } + + /** + * 设置资产列表。 + * + * @param assets 资产列表 + */ + public void setAssets(List assets) { + this.assets = assets == null ? new ArrayList<>() : new ArrayList<>(assets); + } + + /** + * 转换为轻量描述。 + * + * @return Skill 描述 + */ + public SkillDescriptor toDescriptor() { + return new SkillDescriptor(id, name, description, metadata); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java new file mode 100644 index 0000000..e5bfda1 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java @@ -0,0 +1,145 @@ +package com.easyagents.skill.model; + +import java.io.Serializable; + +/** + * Skill 静态资产。 + */ +public class SkillAsset implements Serializable { + + private static final long serialVersionUID = 1L; + + private String path; + private String name; + private String mediaType; + private String contentRef; + private String contentHash; + private long size; + private SkillMetadata metadata = new SkillMetadata(); + + /** + * 获取逻辑路径。 + * + * @return 逻辑路径 + */ + public String getPath() { + return path; + } + + /** + * 设置逻辑路径。 + * + * @param path 逻辑路径 + */ + public void setPath(String path) { + this.path = path; + } + + /** + * 获取名称。 + * + * @return 名称 + */ + public String getName() { + return name; + } + + /** + * 设置名称。 + * + * @param name 名称 + */ + public void setName(String name) { + this.name = name; + } + + /** + * 获取媒体类型。 + * + * @return 媒体类型 + */ + public String getMediaType() { + return mediaType; + } + + /** + * 设置媒体类型。 + * + * @param mediaType 媒体类型 + */ + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + /** + * 获取内容引用。 + * + * @return 内容引用 + */ + public String getContentRef() { + return contentRef; + } + + /** + * 设置内容引用。 + * + * @param contentRef 内容引用 + */ + public void setContentRef(String contentRef) { + this.contentRef = contentRef; + } + + /** + * 获取内容 hash。 + * + * @return 内容 hash + */ + public String getContentHash() { + return contentHash; + } + + /** + * 设置内容 hash。 + * + * @param contentHash 内容 hash + */ + public void setContentHash(String contentHash) { + this.contentHash = contentHash; + } + + /** + * 获取文件大小。 + * + * @return 文件大小 + */ + public long getSize() { + return size; + } + + /** + * 设置文件大小。 + * + * @param size 文件大小 + */ + public void setSize(long size) { + this.size = size; + } + + /** + * 获取元数据。 + * + * @return 元数据 + */ + public SkillMetadata getMetadata() { + return metadata; + } + + /** + * 设置元数据。 + * + * @param metadata 元数据 + */ + public void setMetadata(SkillMetadata metadata) { + this.metadata = metadata == null ? new SkillMetadata() : metadata; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java new file mode 100644 index 0000000..a25df10 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java @@ -0,0 +1,111 @@ +package com.easyagents.skill.model; + +import java.io.Serial; +import java.io.Serializable; + +/** + * Skill 轻量描述。 + */ +public class SkillDescriptor implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private String id; + private String name; + private String description; + private SkillMetadata metadata = new SkillMetadata(); + + /** + * 创建空 Skill 描述。 + */ + public SkillDescriptor() { + } + + /** + * 创建 Skill 描述。 + * + * @param id Skill ID + * @param name Skill 名称 + * @param description Skill 描述 + * @param metadata 元数据 + */ + public SkillDescriptor(String id, String name, String description, SkillMetadata metadata) { + this.id = id; + this.name = name; + this.description = description; + setMetadata(metadata); + } + + /** + * 获取 Skill ID。 + * + * @return Skill ID + */ + public String getId() { + return id; + } + + /** + * 设置 Skill ID。 + * + * @param id Skill ID + */ + public void setId(String id) { + this.id = id; + } + + /** + * 获取名称。 + * + * @return 名称 + */ + public String getName() { + return name; + } + + /** + * 设置名称。 + * + * @param name 名称 + */ + public void setName(String name) { + this.name = name; + } + + /** + * 获取描述。 + * + * @return 描述 + */ + public String getDescription() { + return description; + } + + /** + * 设置描述。 + * + * @param description 描述 + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * 获取元数据。 + * + * @return 元数据 + */ + public SkillMetadata getMetadata() { + return metadata; + } + + /** + * 设置元数据。 + * + * @param metadata 元数据 + */ + public void setMetadata(SkillMetadata metadata) { + this.metadata = metadata == null ? new SkillMetadata() : metadata; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java new file mode 100644 index 0000000..f05e768 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java @@ -0,0 +1,79 @@ +package com.easyagents.skill.model; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Skill 元数据容器。 + */ +public class SkillMetadata implements Serializable { + + private static final long serialVersionUID = 1L; + + private Map values = new LinkedHashMap<>(); + + /** + * 创建空元数据。 + */ + public SkillMetadata() { + } + + /** + * 使用已有键值创建元数据。 + * + * @param values 元数据键值 + */ + public SkillMetadata(Map values) { + setValues(values); + } + + /** + * 获取元数据键值。 + * + * @return 元数据键值 + */ + public Map getValues() { + return values; + } + + /** + * 设置元数据键值。 + * + * @param values 元数据键值 + */ + public void setValues(Map values) { + this.values = values == null ? new LinkedHashMap<>() : new LinkedHashMap<>(values); + } + + /** + * 写入一个元数据键值。 + * + * @param key 元数据键 + * @param value 元数据值 + */ + public void put(String key, Object value) { + if (key != null && !key.isBlank()) { + values.put(key, value); + } + } + + /** + * 读取元数据值。 + * + * @param key 元数据键 + * @return 元数据值 + */ + public Object get(String key) { + return values.get(key); + } + + /** + * 判断是否为空。 + * + * @return 无元数据时为 true + */ + public boolean isEmpty() { + return values.isEmpty(); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java new file mode 100644 index 0000000..c27af7e --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java @@ -0,0 +1,126 @@ +package com.easyagents.skill.model; + +import java.io.Serializable; + +/** + * Skill Markdown 参考文档。 + */ +public class SkillReference implements Serializable { + + private static final long serialVersionUID = 1L; + + private String path; + private String name; + private String content; + private String contentHash; + private long size; + private SkillMetadata metadata = new SkillMetadata(); + + /** + * 获取逻辑路径。 + * + * @return 逻辑路径 + */ + public String getPath() { + return path; + } + + /** + * 设置逻辑路径。 + * + * @param path 逻辑路径 + */ + public void setPath(String path) { + this.path = path; + } + + /** + * 获取名称。 + * + * @return 名称 + */ + public String getName() { + return name; + } + + /** + * 设置名称。 + * + * @param name 名称 + */ + public void setName(String name) { + this.name = name; + } + + /** + * 获取 Markdown 内容。 + * + * @return Markdown 内容 + */ + public String getContent() { + return content; + } + + /** + * 设置 Markdown 内容。 + * + * @param content Markdown 内容 + */ + public void setContent(String content) { + this.content = content; + } + + /** + * 获取内容 hash。 + * + * @return 内容 hash + */ + public String getContentHash() { + return contentHash; + } + + /** + * 设置内容 hash。 + * + * @param contentHash 内容 hash + */ + public void setContentHash(String contentHash) { + this.contentHash = contentHash; + } + + /** + * 获取文件大小。 + * + * @return 文件大小 + */ + public long getSize() { + return size; + } + + /** + * 设置文件大小。 + * + * @param size 文件大小 + */ + public void setSize(long size) { + this.size = size; + } + + /** + * 获取元数据。 + * + * @return 元数据 + */ + public SkillMetadata getMetadata() { + return metadata; + } + + /** + * 设置元数据。 + * + * @param metadata 元数据 + */ + public void setMetadata(SkillMetadata metadata) { + this.metadata = metadata == null ? new SkillMetadata() : metadata; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java new file mode 100644 index 0000000..edab344 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java @@ -0,0 +1,126 @@ +package com.easyagents.skill.model; + +import java.io.Serializable; + +/** + * Skill 脚本源码。 + */ +public class SkillScript implements Serializable { + + private static final long serialVersionUID = 1L; + + private String path; + private SkillScriptLanguage language = SkillScriptLanguage.UNKNOWN; + private String content; + private String contentHash; + private long size; + private SkillMetadata metadata = new SkillMetadata(); + + /** + * 获取逻辑路径。 + * + * @return 逻辑路径 + */ + public String getPath() { + return path; + } + + /** + * 设置逻辑路径。 + * + * @param path 逻辑路径 + */ + public void setPath(String path) { + this.path = path; + } + + /** + * 获取脚本语言。 + * + * @return 脚本语言 + */ + public SkillScriptLanguage getLanguage() { + return language; + } + + /** + * 设置脚本语言。 + * + * @param language 脚本语言 + */ + public void setLanguage(SkillScriptLanguage language) { + this.language = language == null ? SkillScriptLanguage.UNKNOWN : language; + } + + /** + * 获取脚本源码。 + * + * @return 脚本源码 + */ + public String getContent() { + return content; + } + + /** + * 设置脚本源码。 + * + * @param content 脚本源码 + */ + public void setContent(String content) { + this.content = content; + } + + /** + * 获取内容 hash。 + * + * @return 内容 hash + */ + public String getContentHash() { + return contentHash; + } + + /** + * 设置内容 hash。 + * + * @param contentHash 内容 hash + */ + public void setContentHash(String contentHash) { + this.contentHash = contentHash; + } + + /** + * 获取文件大小。 + * + * @return 文件大小 + */ + public long getSize() { + return size; + } + + /** + * 设置文件大小。 + * + * @param size 文件大小 + */ + public void setSize(long size) { + this.size = size; + } + + /** + * 获取元数据。 + * + * @return 元数据 + */ + public SkillMetadata getMetadata() { + return metadata; + } + + /** + * 设置元数据。 + * + * @param metadata 元数据 + */ + public void setMetadata(SkillMetadata metadata) { + this.metadata = metadata == null ? new SkillMetadata() : metadata; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java new file mode 100644 index 0000000..c9ce329 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java @@ -0,0 +1,50 @@ +package com.easyagents.skill.model; + +/** + * Skill 脚本语言。 + */ +public enum SkillScriptLanguage { + + /** + * Python 脚本。 + */ + PYTHON, + + /** + * JavaScript 脚本。 + */ + JAVASCRIPT, + + /** + * Shell 脚本。 + */ + SHELL, + + /** + * 未知脚本语言。 + */ + UNKNOWN; + + /** + * 按脚本路径识别语言。 + * + * @param path 脚本逻辑路径 + * @return 脚本语言 + */ + public static SkillScriptLanguage fromPath(String path) { + if (path == null) { + return UNKNOWN; + } + String lower = path.toLowerCase(); + if (lower.endsWith(".py")) { + return PYTHON; + } + if (lower.endsWith(".js")) { + return JAVASCRIPT; + } + if (lower.endsWith(".sh")) { + return SHELL; + } + return UNKNOWN; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java new file mode 100644 index 0000000..03552ee --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java @@ -0,0 +1,58 @@ +package com.easyagents.skill.repository; + +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillDescriptor; + +import java.util.List; +import java.util.Optional; + +/** + * Skill 聚合存储接口。 + */ +public interface SkillRepository { + + /** + * 保存 Skill。 + * + * @param skill Skill 聚合 + */ + void save(Skill skill); + + /** + * 获取完整 Skill。 + * + * @param skillId Skill ID + * @return Skill 聚合 + */ + Optional get(String skillId); + + /** + * 获取 Skill 描述。 + * + * @param skillId Skill ID + * @return Skill 描述 + */ + Optional getDescriptor(String skillId); + + /** + * 列出 Skill 描述。 + * + * @return Skill 描述列表 + */ + List listDescriptors(); + + /** + * 删除 Skill。 + * + * @param skillId Skill ID + */ + void delete(String skillId); + + /** + * 判断 Skill 是否存在。 + * + * @param skillId Skill ID + * @return 存在时为 true + */ + boolean exists(String skillId); +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java new file mode 100644 index 0000000..60795d1 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java @@ -0,0 +1,176 @@ +package com.easyagents.skill.repository.memory; + +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.model.*; +import com.easyagents.skill.repository.SkillRepository; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * 基于内存的 Skill 聚合仓储实现。 + */ +public class InMemorySkillRepository implements SkillRepository { + + private final ConcurrentMap skills = new ConcurrentHashMap<>(); + + /** + * 保存 Skill 聚合。 + * + * @param skill Skill 聚合 + */ + @Override + public void save(Skill skill) { + if (skill == null || isBlank(skill.getId())) { + throw new SkillValidationException("Skill id is required."); + } + skills.put(skill.getId(), copySkill(skill)); + } + + /** + * 获取完整 Skill。 + * + * @param skillId Skill ID + * @return Skill 聚合 + */ + @Override + public Optional get(String skillId) { + Skill skill = skills.get(skillId); + return skill == null ? Optional.empty() : Optional.of(copySkill(skill)); + } + + /** + * 获取 Skill 描述。 + * + * @param skillId Skill ID + * @return Skill 描述 + */ + @Override + public Optional getDescriptor(String skillId) { + Skill skill = skills.get(skillId); + return skill == null ? Optional.empty() : Optional.of(copyDescriptor(skill.toDescriptor())); + } + + /** + * 列出 Skill 描述。 + * + * @return Skill 描述列表 + */ + @Override + public List listDescriptors() { + List descriptors = new ArrayList<>(); + for (Skill skill : skills.values()) { + descriptors.add(copyDescriptor(skill.toDescriptor())); + } + return descriptors; + } + + /** + * 删除 Skill。 + * + * @param skillId Skill ID + */ + @Override + public void delete(String skillId) { + skills.remove(skillId); + } + + /** + * 判断 Skill 是否存在。 + * + * @param skillId Skill ID + * @return 存在时为 true + */ + @Override + public boolean exists(String skillId) { + return skills.containsKey(skillId); + } + + private static Skill copySkill(Skill source) { + Skill target = new Skill(); + target.setId(source.getId()); + target.setName(source.getName()); + target.setDescription(source.getDescription()); + target.setMetadata(copyMetadata(source.getMetadata())); + target.setSkillContent(source.getSkillContent()); + target.setReferences(copyReferences(source.getReferences())); + target.setScripts(copyScripts(source.getScripts())); + target.setAssets(copyAssets(source.getAssets())); + return target; + } + + private static List copyReferences(List sources) { + List targets = new ArrayList<>(); + if (sources == null) { + return targets; + } + for (SkillReference source : sources) { + SkillReference target = new SkillReference(); + target.setPath(source.getPath()); + target.setName(source.getName()); + target.setContent(source.getContent()); + target.setContentHash(source.getContentHash()); + target.setSize(source.getSize()); + target.setMetadata(copyMetadata(source.getMetadata())); + targets.add(target); + } + return targets; + } + + private static List copyScripts(List sources) { + List targets = new ArrayList<>(); + if (sources == null) { + return targets; + } + for (SkillScript source : sources) { + SkillScript target = new SkillScript(); + target.setPath(source.getPath()); + target.setLanguage(source.getLanguage()); + target.setContent(source.getContent()); + target.setContentHash(source.getContentHash()); + target.setSize(source.getSize()); + target.setMetadata(copyMetadata(source.getMetadata())); + targets.add(target); + } + return targets; + } + + private static List copyAssets(List sources) { + List targets = new ArrayList<>(); + if (sources == null) { + return targets; + } + for (SkillAsset source : sources) { + SkillAsset target = new SkillAsset(); + target.setPath(source.getPath()); + target.setName(source.getName()); + target.setMediaType(source.getMediaType()); + target.setContentRef(source.getContentRef()); + target.setContentHash(source.getContentHash()); + target.setSize(source.getSize()); + target.setMetadata(copyMetadata(source.getMetadata())); + targets.add(target); + } + return targets; + } + + private static SkillDescriptor copyDescriptor(SkillDescriptor source) { + return new SkillDescriptor( + source.getId(), + source.getName(), + source.getDescription(), + copyMetadata(source.getMetadata()) + ); + } + + private static SkillMetadata copyMetadata(SkillMetadata source) { + return source == null ? new SkillMetadata() : new SkillMetadata(source.getValues()); + } + + private static boolean isBlank(String value) { + return value == null || value.isBlank(); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java new file mode 100644 index 0000000..e712c4e --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java @@ -0,0 +1,41 @@ +package com.easyagents.skill.store; + +import java.io.InputStream; + +/** + * Skill 二进制内容存储接口。 + */ +public interface SkillContentStore { + + /** + * 保存内容并返回内容引用。 + * + * @param bytes 内容字节 + * @return 内容引用 + */ + String put(byte[] bytes); + + /** + * 打开内容流。 + * + * @param contentRef 内容引用 + * @return 内容流 + */ + InputStream open(String contentRef); + + /** + * 读取全部内容。 + * + * @param contentRef 内容引用 + * @return 内容字节 + */ + byte[] readAllBytes(String contentRef); + + /** + * 判断内容是否存在。 + * + * @param contentRef 内容引用 + * @return 存在时为 true + */ + boolean exists(String contentRef); +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java new file mode 100644 index 0000000..98632aa --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java @@ -0,0 +1,70 @@ +package com.easyagents.skill.store.memory; + +import com.easyagents.skill.exception.SkillException; +import com.easyagents.skill.store.SkillContentStore; +import com.easyagents.skill.util.SkillHashes; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * 基于内存的 Skill 二进制内容存储。 + */ +public class InMemorySkillContentStore implements SkillContentStore { + + private final ConcurrentMap contents = new ConcurrentHashMap<>(); + + /** + * 保存内容并返回内容引用。 + * + * @param bytes 内容字节 + * @return 内容引用 + */ + @Override + public String put(byte[] bytes) { + byte[] safeBytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); + String contentRef = SkillHashes.sha256Ref(safeBytes); + contents.putIfAbsent(contentRef, safeBytes); + return contentRef; + } + + /** + * 打开内容流。 + * + * @param contentRef 内容引用 + * @return 内容流 + */ + @Override + public InputStream open(String contentRef) { + return new ByteArrayInputStream(readAllBytes(contentRef)); + } + + /** + * 读取全部内容。 + * + * @param contentRef 内容引用 + * @return 内容字节 + */ + @Override + public byte[] readAllBytes(String contentRef) { + byte[] bytes = contents.get(contentRef); + if (bytes == null) { + throw new SkillException("Skill content does not exist: " + contentRef); + } + return Arrays.copyOf(bytes, bytes.length); + } + + /** + * 判断内容是否存在。 + * + * @param contentRef 内容引用 + * @return 存在时为 true + */ + @Override + public boolean exists(String contentRef) { + return contents.containsKey(contentRef); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java new file mode 100644 index 0000000..85e748e --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java @@ -0,0 +1,89 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.exception.SkillValidationException; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * SKILL.md frontmatter 解析工具。 + */ +public final class SkillFrontmatter { + + private SkillFrontmatter() { + } + + /** + * 解析 SKILL.md 开头的 frontmatter。 + * + * @param content SKILL.md 原始内容 + * @return frontmatter 键值 + */ + public static Map parse(String content) { + if (content == null || content.isBlank()) { + throw new SkillValidationException("SKILL.md content is required."); + } + String[] lines = content.split("\\R", -1); + if (lines.length == 0 || !"---".equals(lines[0])) { + throw new SkillValidationException("SKILL.md must start with frontmatter."); + } + + Map values = new LinkedHashMap<>(); + boolean closed = false; + for (int i = 1; i < lines.length; i++) { + String line = lines[i]; + if ("---".equals(line)) { + closed = true; + break; + } + if (line.isBlank()) { + continue; + } + if (Character.isWhitespace(line.charAt(0))) { + throw new SkillValidationException("Nested frontmatter is not supported."); + } + parseScalarLine(line, values); + } + if (!closed) { + throw new SkillValidationException("SKILL.md frontmatter is not closed."); + } + if (isBlank(values.get("name"))) { + throw new SkillValidationException("SKILL.md frontmatter name is required."); + } + if (isBlank(values.get("description"))) { + throw new SkillValidationException("SKILL.md frontmatter description is required."); + } + return values; + } + + private static void parseScalarLine(String line, Map values) { + int colonIndex = line.indexOf(':'); + if (colonIndex <= 0) { + throw new SkillValidationException("Only single-line key: value frontmatter is supported."); + } + String key = line.substring(0, colonIndex).trim(); + String value = stripQuotes(line.substring(colonIndex + 1).trim()); + if (key.isBlank()) { + throw new SkillValidationException("Frontmatter key cannot be blank."); + } + if (value.isBlank()) { + throw new SkillValidationException("Frontmatter value cannot be blank: " + key); + } + values.put(key, value); + } + + private static String stripQuotes(String value) { + if (value.length() >= 2) { + boolean doubleQuoted = value.startsWith("\"") && value.endsWith("\""); + boolean singleQuoted = value.startsWith("'") && value.endsWith("'"); + if (doubleQuoted || singleQuoted) { + return value.substring(1, value.length() - 1); + } + } + return value; + } + + private static boolean isBlank(Object value) { + return value == null || value.toString().isBlank(); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java new file mode 100644 index 0000000..cefd9e1 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java @@ -0,0 +1,47 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.exception.SkillException; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Skill 内容 hash 工具。 + */ +public final class SkillHashes { + + private static final String SHA_256 = "SHA-256"; + + private SkillHashes() { + } + + /** + * 计算 SHA-256 内容引用。 + * + * @param bytes 内容字节 + * @return sha256 内容引用 + */ + public static String sha256Ref(byte[] bytes) { + return "sha256:" + sha256Hex(bytes); + } + + /** + * 计算 SHA-256 十六进制值。 + * + * @param bytes 内容字节 + * @return 十六进制 hash + */ + public static String sha256Hex(byte[] bytes) { + try { + MessageDigest digest = MessageDigest.getInstance(SHA_256); + byte[] hashed = digest.digest(bytes == null ? new byte[0] : bytes); + StringBuilder builder = new StringBuilder(hashed.length * 2); + for (byte item : hashed) { + builder.append(String.format("%02x", item)); + } + return builder.toString(); + } catch (NoSuchAlgorithmException e) { + throw new SkillException("SHA-256 algorithm is unavailable.", e); + } + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java new file mode 100644 index 0000000..a2c61ba --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java @@ -0,0 +1,118 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.exception.SkillValidationException; + +import java.util.Locale; + +/** + * Skill 逻辑路径工具。 + */ +public final class SkillPaths { + + /** + * Skill 主文件名。 + */ + public static final String SKILL_FILE = "SKILL.md"; + + /** + * reference 顶级目录。 + */ + public static final String REFERENCES_DIR = "references"; + + /** + * script 顶级目录。 + */ + public static final String SCRIPTS_DIR = "scripts"; + + /** + * asset 顶级目录。 + */ + public static final String ASSETS_DIR = "assets"; + + private SkillPaths() { + } + + /** + * 规范化逻辑路径。 + * + * @param path 原始路径 + * @return 规范化后的路径 + */ + public static String normalize(String path) { + if (path == null) { + throw new SkillValidationException("Skill path is required."); + } + String normalized = path.replace('\\', '/').trim(); + while (normalized.startsWith("./")) { + normalized = normalized.substring(2); + } + if (normalized.isEmpty()) { + throw new SkillValidationException("Skill path cannot be empty."); + } + if (normalized.startsWith("/") || normalized.matches("^[A-Za-z]:.*")) { + throw new SkillValidationException("Absolute skill path is not allowed: " + path); + } + String[] segments = normalized.split("/"); + for (String segment : segments) { + if (segment.isEmpty() || ".".equals(segment) || "..".equals(segment)) { + throw new SkillValidationException("Unsafe skill path is not allowed: " + path); + } + if (segment.startsWith(".")) { + throw new SkillValidationException("Hidden skill path is not allowed: " + path); + } + } + return normalized; + } + + /** + * 获取一级目录或文件名。 + * + * @param path 逻辑路径 + * @return 一级路径段 + */ + public static String firstSegment(String path) { + String normalized = normalize(path); + int index = normalized.indexOf('/'); + return index < 0 ? normalized : normalized.substring(0, index); + } + + /** + * 获取文件名。 + * + * @param path 逻辑路径 + * @return 文件名 + */ + public static String fileName(String path) { + String normalized = normalize(path); + int index = normalized.lastIndexOf('/'); + return index < 0 ? normalized : normalized.substring(index + 1); + } + + /** + * 判断扩展名是否匹配。 + * + * @param path 逻辑路径 + * @param extension 扩展名 + * @return 匹配时为 true + */ + public static boolean hasExtension(String path, String extension) { + return normalize(path).toLowerCase(Locale.ROOT).endsWith(extension.toLowerCase(Locale.ROOT)); + } + + /** + * 判断是否为系统文件路径。 + * + * @param path 原始路径 + * @return 系统文件时为 true + */ + public static boolean isIgnoredSystemPath(String path) { + if (path == null) { + return true; + } + String normalized = path.replace('\\', '/'); + return normalized.startsWith("__MACOSX/") + || normalized.contains("/__MACOSX/") + || normalized.endsWith("/.DS_Store") + || ".DS_Store".equals(normalized); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java new file mode 100644 index 0000000..b0ef7dc --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java @@ -0,0 +1,16 @@ +package com.easyagents.skill.validation; + +import com.easyagents.skill.model.Skill; + +/** + * Skill 校验接口。 + */ +public interface SkillValidator { + + /** + * 校验 Skill 聚合。 + * + * @param skill Skill 聚合 + */ + void validate(Skill skill); +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java new file mode 100644 index 0000000..06fc481 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java @@ -0,0 +1,156 @@ +package com.easyagents.skill.validation.defaults; + +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.model.*; +import com.easyagents.skill.util.SkillFrontmatter; +import com.easyagents.skill.util.SkillHashes; +import com.easyagents.skill.util.SkillPaths; +import com.easyagents.skill.validation.SkillValidator; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 默认 Skill 聚合校验器。 + */ +public class DefaultSkillValidator implements SkillValidator { + + /** + * 校验 Skill 聚合。 + * + * @param skill Skill 聚合 + */ + @Override + public void validate(Skill skill) { + if (skill == null) { + throw new SkillValidationException("Skill is required."); + } + requireText(skill.getId(), "Skill id is required."); + requireText(skill.getName(), "Skill name is required."); + requireText(skill.getDescription(), "Skill description is required."); + requireText(skill.getSkillContent(), "SKILL.md content is required."); + validateSkillFrontmatter(skill); + + Set paths = new HashSet<>(); + paths.add(SkillPaths.SKILL_FILE); + validateReferences(skill.getReferences(), paths); + validateScripts(skill.getScripts(), paths); + validateAssets(skill.getAssets(), paths); + } + + private static void validateReferences(List references, Set paths) { + if (references == null) { + return; + } + for (SkillReference reference : references) { + if (reference == null) { + throw new SkillValidationException("Skill reference cannot be null."); + } + String path = validateFilePath(reference.getPath(), SkillPaths.REFERENCES_DIR, paths); + if (!SkillPaths.hasExtension(path, ".md")) { + throw new SkillValidationException("Skill reference must be a markdown file: " + path); + } + requireContent(reference.getContent(), "Skill reference content is required: " + path); + validateTextHash(path, reference.getContent(), reference.getContentHash()); + validateSize(reference.getSize(), path); + } + } + + private static void validateScripts(List scripts, Set paths) { + if (scripts == null) { + return; + } + for (SkillScript script : scripts) { + if (script == null) { + throw new SkillValidationException("Skill script cannot be null."); + } + String path = validateFilePath(script.getPath(), SkillPaths.SCRIPTS_DIR, paths); + if (SkillScriptLanguage.fromPath(path) == SkillScriptLanguage.UNKNOWN) { + throw new SkillValidationException("Unsupported skill script extension: " + path); + } + if (script.getLanguage() == SkillScriptLanguage.UNKNOWN + || script.getLanguage() != SkillScriptLanguage.fromPath(path)) { + throw new SkillValidationException("Skill script language does not match path: " + path); + } + requireContent(script.getContent(), "Skill script content is required: " + path); + validateTextHash(path, script.getContent(), script.getContentHash()); + validateSize(script.getSize(), path); + } + } + + private static void validateAssets(List assets, Set paths) { + if (assets == null) { + return; + } + for (SkillAsset asset : assets) { + if (asset == null) { + throw new SkillValidationException("Skill asset cannot be null."); + } + String path = validateFilePath(asset.getPath(), SkillPaths.ASSETS_DIR, paths); + requireText(asset.getName(), "Skill asset name is required: " + path); + requireText(asset.getMediaType(), "Skill asset media type is required: " + path); + requireText(asset.getContentRef(), "Skill asset content ref is required: " + path); + requireText(asset.getContentHash(), "Skill asset content hash is required: " + path); + if (!asset.getContentRef().equals("sha256:" + asset.getContentHash())) { + throw new SkillValidationException("Skill asset content ref does not match hash: " + path); + } + validateSize(asset.getSize(), path); + } + } + + private static void validateSkillFrontmatter(Skill skill) { + Map values = SkillFrontmatter.parse(skill.getSkillContent()); + if (!skill.getName().equals(values.get("name").toString())) { + throw new SkillValidationException("Skill name must match SKILL.md frontmatter."); + } + if (!skill.getDescription().equals(values.get("description").toString())) { + throw new SkillValidationException("Skill description must match SKILL.md frontmatter."); + } + if (skill.getMetadata() == null || !skill.getMetadata().getValues().equals(values)) { + throw new SkillValidationException("Skill metadata must match SKILL.md frontmatter."); + } + } + + private static String validateFilePath(String path, String expectedTopDir, Set paths) { + String normalized = SkillPaths.normalize(path); + if (!expectedTopDir.equals(SkillPaths.firstSegment(normalized))) { + throw new SkillValidationException("Skill file must be under " + expectedTopDir + "/: " + normalized); + } + if (normalized.indexOf('/') < 0 || normalized.endsWith("/")) { + throw new SkillValidationException("Skill file path must include file name: " + normalized); + } + if (!paths.add(normalized)) { + throw new SkillValidationException("Duplicate skill file path: " + normalized); + } + return normalized; + } + + private static void validateTextHash(String path, String content, String contentHash) { + requireText(contentHash, "Skill file content hash is required: " + path); + String actualHash = SkillHashes.sha256Hex(content.getBytes(StandardCharsets.UTF_8)); + if (!actualHash.equals(contentHash)) { + throw new SkillValidationException("Skill file content hash does not match: " + path); + } + } + + private static void validateSize(long size, String path) { + if (size < 0) { + throw new SkillValidationException("Skill file size cannot be negative: " + path); + } + } + + private static void requireText(String value, String message) { + if (value == null || value.isBlank()) { + throw new SkillValidationException(message); + } + } + + private static void requireContent(String value, String message) { + if (value == null) { + throw new SkillValidationException(message); + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java new file mode 100644 index 0000000..50831bc --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java @@ -0,0 +1,275 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.exception.SkillPackageException; +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.store.SkillContentStore; +import com.easyagents.skill.store.memory.InMemorySkillContentStore; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * ZipSkillPackageCodec 单元测试。 + */ +public class ZipSkillPackageCodecTest { + + /** + * 导入包含一个 Skill 文件夹的 zip。 + */ + @Test + public void importOneSkillFolder() { + List skills = importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/references/rules/a.md", "# Rule", + "skill-a/scripts/run.py", "print('ok')", + "skill-a/assets/images/logo.png", "png-data" + )); + + Assert.assertEquals(1, skills.size()); + Skill skill = skills.get(0); + Assert.assertEquals("skill-a", skill.getId()); + Assert.assertEquals("Skill A", skill.getName()); + Assert.assertEquals("Desc A", skill.getDescription()); + Assert.assertEquals(1, skill.getReferences().size()); + Assert.assertEquals("references/rules/a.md", skill.getReferences().get(0).getPath()); + Assert.assertEquals(1, skill.getScripts().size()); + Assert.assertEquals(1, skill.getAssets().size()); + Assert.assertEquals("assets/images/logo.png", skill.getAssets().get(0).getPath()); + } + + /** + * 导入包含多个 Skill 文件夹的 zip。 + */ + @Test + public void importMultipleSkillFolders() { + List skills = importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-b/SKILL.md", skillMd("Skill B", "Desc B") + )); + + Assert.assertEquals(2, skills.size()); + Assert.assertEquals("skill-a", skills.get(0).getId()); + Assert.assertEquals("skill-b", skills.get(1).getId()); + } + + /** + * 拒绝 zip 根目录直接包含 SKILL.md。 + */ + @Test(expected = SkillPackageException.class) + public void rejectRootSkillMd() { + importZip(files("SKILL.md", skillMd("Skill", "Desc"))); + } + + /** + * 拒绝未知顶级目录。 + */ + @Test(expected = SkillPackageException.class) + public void rejectUnknownTopLevelDirectory() { + importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/unknown/a.md", "# Unknown" + )); + } + + /** + * 拒绝 references 下的非 md 文件。 + */ + @Test(expected = SkillPackageException.class) + public void rejectNonMarkdownReference() { + importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/references/a.txt", "text" + )); + } + + /** + * 拒绝不支持的脚本扩展名。 + */ + @Test(expected = SkillPackageException.class) + public void rejectUnsupportedScript() { + importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/scripts/run.rb", "puts 'ok'" + )); + } + + /** + * assets 允许任意扩展名。 + */ + @Test + public void allowArbitraryAssetExtension() { + List skills = importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/assets/model.custom", "asset" + )); + + Assert.assertEquals(1, skills.get(0).getAssets().size()); + Assert.assertEquals("application/octet-stream", skills.get(0).getAssets().get(0).getMediaType()); + } + + /** + * 忽略 macOS 系统文件和空目录。 + */ + @Test + public void ignoreSystemFilesAndEmptyDirectories() { + Map files = files( + "__MACOSX/._x", "ignored", + "skill-a/.DS_Store", "ignored", + "skill-a/SKILL.md", skillMd("Skill A", "Desc A") + ); + List skills = new ZipSkillPackageCodec().importZip(new ByteArrayInputStream(zip(files, "skill-a/references/"))); + + Assert.assertEquals(1, skills.size()); + Assert.assertEquals("skill-a", skills.get(0).getId()); + } + + /** + * 嵌套 references 和 assets 路径正常导入。 + */ + @Test + public void allowNestedManagedDirectories() { + List skills = importZip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/references/rules/a.md", "# Nested", + "skill-a/assets/images/logo.png", "logo" + )); + + Assert.assertEquals("references/rules/a.md", skills.get(0).getReferences().get(0).getPath()); + Assert.assertEquals("assets/images/logo.png", skills.get(0).getAssets().get(0).getPath()); + } + + /** + * 资产字节写入内容存储。 + */ + @Test + public void storeAssetContent() { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + List skills = new ZipSkillPackageCodec(store).importZip(new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/assets/data.bin", "abc" + )))); + + String contentRef = skills.get(0).getAssets().get(0).getContentRef(); + Assert.assertTrue(store.exists(contentRef)); + Assert.assertArrayEquals("abc".getBytes(StandardCharsets.UTF_8), store.readAllBytes(contentRef)); + } + + /** + * 导入失败时不写入资产内容。 + */ + @Test + public void failedImportDoesNotWriteAssetContent() { + CountingContentStore store = new CountingContentStore(); + + try { + new ZipSkillPackageCodec(store).importZip(new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/assets/data.bin", "abc", + "skill-a/unknown/a.md", "# Unknown" + )))); + Assert.fail("Import should fail."); + } catch (SkillPackageException expected) { + Assert.assertEquals(0, store.putCount); + } + } + + /** + * 多 Skill 包中后续 Skill 失败时,不提前写入前面 Skill 的资产。 + */ + @Test + public void failedMultiSkillImportDoesNotWriteEarlierAssetContent() { + CountingContentStore store = new CountingContentStore(); + + try { + new ZipSkillPackageCodec(store).importZip(new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), + "skill-a/assets/data.bin", "abc", + "skill-b/references/a.md", "# Missing SKILL.md" + )))); + Assert.fail("Import should fail."); + } catch (SkillPackageException expected) { + Assert.assertEquals(0, store.putCount); + } + } + + /** + * 拒绝嵌套 frontmatter。 + */ + @Test(expected = SkillPackageException.class) + public void rejectNestedFrontmatter() { + importZip(files( + "skill-a/SKILL.md", "---\nname: Skill A\ndescription: Desc A\nconfig:\n mode: strict\n---\n# Skill A\n" + )); + } + + private static List importZip(Map files) { + return new ZipSkillPackageCodec().importZip(new ByteArrayInputStream(zip(files))); + } + + private static Map files(String... keyValues) { + Map files = new LinkedHashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + files.put(keyValues[i], keyValues[i + 1]); + } + return files; + } + + private static byte[] zip(Map files, String... directories) { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(bytes)) { + for (String directory : directories) { + zipOutputStream.putNextEntry(new ZipEntry(directory)); + zipOutputStream.closeEntry(); + } + for (Map.Entry file : files.entrySet()) { + zipOutputStream.putNextEntry(new ZipEntry(file.getKey())); + zipOutputStream.write(file.getValue().getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + } + return bytes.toByteArray(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static String skillMd(String name, String description) { + return "---\nname: " + name + "\ndescription: " + description + "\n---\n# " + name + "\n"; + } + + private static final class CountingContentStore implements SkillContentStore { + + private int putCount; + + @Override + public String put(byte[] bytes) { + putCount++; + return "sha256:" + com.easyagents.skill.util.SkillHashes.sha256Hex(bytes); + } + + @Override + public InputStream open(String contentRef) { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] readAllBytes(String contentRef) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean exists(String contentRef) { + return false; + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java new file mode 100644 index 0000000..fa3a550 --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java @@ -0,0 +1,83 @@ +package com.easyagents.skill.repository.memory; + +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillDescriptor; +import com.easyagents.skill.model.SkillReference; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +/** + * InMemorySkillRepository 单元测试。 + */ +public class InMemorySkillRepositoryTest { + + /** + * 覆盖保存、读取、描述、列表、删除和存在性判断。 + */ + @Test + public void crudSkill() { + InMemorySkillRepository repository = new InMemorySkillRepository(); + Skill skill = skill(); + + repository.save(skill); + + Assert.assertTrue(repository.exists("skill-a")); + Assert.assertTrue(repository.get("skill-a").isPresent()); + Assert.assertTrue(repository.getDescriptor("skill-a").isPresent()); + Assert.assertEquals(1, repository.listDescriptors().size()); + + repository.delete("skill-a"); + + Assert.assertFalse(repository.exists("skill-a")); + Assert.assertFalse(repository.get("skill-a").isPresent()); + } + + /** + * descriptor 不携带 references/scripts/assets 内容。 + */ + @Test + public void descriptorDoesNotExposeResourceContent() { + InMemorySkillRepository repository = new InMemorySkillRepository(); + repository.save(skill()); + + Optional descriptor = repository.getDescriptor("skill-a"); + + Assert.assertTrue(descriptor.isPresent()); + Assert.assertEquals("skill-a", descriptor.get().getId()); + Assert.assertEquals("Skill A", descriptor.get().getName()); + } + + /** + * 读取返回副本,避免外部修改仓储内部状态。 + */ + @Test + public void getReturnsCopy() { + InMemorySkillRepository repository = new InMemorySkillRepository(); + repository.save(skill()); + + Skill loaded = repository.get("skill-a").get(); + loaded.setName("Changed"); + loaded.getReferences().get(0).setContent("changed"); + + Skill reloaded = repository.get("skill-a").get(); + Assert.assertEquals("Skill A", reloaded.getName()); + Assert.assertEquals("# A", reloaded.getReferences().get(0).getContent()); + } + + private static Skill skill() { + Skill skill = new Skill(); + skill.setId("skill-a"); + skill.setName("Skill A"); + skill.setDescription("Desc A"); + skill.setSkillContent("---\nname: Skill A\ndescription: Desc A\n---\n# Skill A\n"); + + SkillReference reference = new SkillReference(); + reference.setPath("references/a.md"); + reference.setName("a.md"); + reference.setContent("# A"); + skill.getReferences().add(reference); + return skill; + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java new file mode 100644 index 0000000..1050cbe --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java @@ -0,0 +1,46 @@ +package com.easyagents.skill.store.memory; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * InMemorySkillContentStore 单元测试。 + */ +public class InMemorySkillContentStoreTest { + + /** + * put 返回 sha256 内容引用,且相同内容引用相同。 + */ + @Test + public void putReturnsStableSha256Ref() { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + + String firstRef = store.put("abc".getBytes(StandardCharsets.UTF_8)); + String secondRef = store.put("abc".getBytes(StandardCharsets.UTF_8)); + + Assert.assertTrue(firstRef.startsWith("sha256:")); + Assert.assertEquals(firstRef, secondRef); + } + + /** + * open、readAllBytes 和 exists 正常工作。 + * + * @throws Exception 读取流失败时抛出 + */ + @Test + public void openReadAndExists() throws Exception { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); + + String contentRef = store.put(bytes); + + Assert.assertTrue(store.exists(contentRef)); + Assert.assertArrayEquals(bytes, store.readAllBytes(contentRef)); + try (InputStream inputStream = store.open(contentRef)) { + Assert.assertArrayEquals(bytes, inputStream.readAllBytes()); + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java new file mode 100644 index 0000000..fdaab0a --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java @@ -0,0 +1,193 @@ +package com.easyagents.skill.validation.defaults; + +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.factory.SkillFactory; +import com.easyagents.skill.model.*; +import com.easyagents.skill.util.SkillHashes; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +/** + * DefaultSkillValidator 单元测试。 + */ +public class DefaultSkillValidatorTest { + + private final DefaultSkillValidator validator = new DefaultSkillValidator(); + + /** + * 缺失 SKILL.md 内容时失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectMissingSkillContent() { + Skill skill = validSkill(); + skill.setSkillContent(null); + + validator.validate(skill); + } + + /** + * 缺失名称时失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectMissingName() { + Skill skill = validSkill(); + skill.setName(""); + + validator.validate(skill); + } + + /** + * 缺失描述时失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectMissingDescription() { + Skill skill = validSkill(); + skill.setDescription(""); + + validator.validate(skill); + } + + /** + * 字段名称必须与 SKILL.md frontmatter 一致。 + */ + @Test(expected = SkillValidationException.class) + public void rejectNameMismatchWithFrontmatter() { + Skill skill = validSkill(); + skill.setName("Changed"); + + validator.validate(skill); + } + + /** + * 元数据必须与 SKILL.md frontmatter 一致。 + */ + @Test(expected = SkillValidationException.class) + public void rejectMetadataMismatchWithFrontmatter() { + Skill skill = validSkill(); + skill.getMetadata().put("extra", "x"); + + validator.validate(skill); + } + + /** + * 拒绝嵌套 frontmatter。 + */ + @Test(expected = SkillValidationException.class) + public void rejectNestedFrontmatter() { + Skill skill = validSkill(); + skill.setSkillContent("---\nname: Skill A\ndescription: Desc A\nconfig:\n mode: strict\n---\n# Skill A\n"); + + validator.validate(skill); + } + + /** + * 绝对路径失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectAbsolutePath() { + Skill skill = validSkill(); + SkillReference reference = reference("/references/a.md", "# A"); + skill.getReferences().add(reference); + + validator.validate(skill); + } + + /** + * ../ 路径失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectParentPath() { + Skill skill = validSkill(); + skill.getReferences().add(reference("references/../a.md", "# A")); + + validator.validate(skill); + } + + /** + * 重复路径失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectDuplicatePath() { + Skill skill = validSkill(); + skill.getReferences().add(reference("references/a.md", "# A")); + skill.getReferences().add(reference("references/a.md", "# B")); + + validator.validate(skill); + } + + /** + * 不因为文件大小较大而失败。 + */ + @Test + public void allowLargeRecordedSize() { + Skill skill = validSkill(); + SkillReference reference = reference("references/a.md", "# A"); + reference.setSize(Long.MAX_VALUE); + skill.getReferences().add(reference); + + validator.validate(skill); + } + + /** + * 允许空内容文件,只校验 hash 和 size 记录。 + */ + @Test + public void allowEmptyReferenceAndScriptContent() { + Skill skill = validSkill(); + skill.getReferences().add(reference("references/empty.md", "")); + skill.getScripts().add(script("scripts/empty.sh", "")); + + validator.validate(skill); + } + + /** + * 校验通过完整 Skill。 + */ + @Test + public void validateCompleteSkill() { + Skill skill = validSkill(); + skill.getReferences().add(reference("references/a.md", "# A")); + skill.getScripts().add(script("scripts/run.sh", "echo ok")); + skill.getAssets().add(asset("assets/a.bin", "abc")); + + validator.validate(skill); + } + + private static Skill validSkill() { + return SkillFactory.create("skill-a", "---\nname: Skill A\ndescription: Desc A\n---\n# Skill A\n"); + } + + private static SkillReference reference(String path, String content) { + SkillReference reference = new SkillReference(); + reference.setPath(path); + reference.setName("a.md"); + reference.setContent(content); + reference.setContentHash(SkillHashes.sha256Hex(content.getBytes(StandardCharsets.UTF_8))); + reference.setSize(content.getBytes(StandardCharsets.UTF_8).length); + return reference; + } + + private static SkillScript script(String path, String content) { + SkillScript script = new SkillScript(); + script.setPath(path); + script.setLanguage(SkillScriptLanguage.fromPath(path)); + script.setContent(content); + script.setContentHash(SkillHashes.sha256Hex(content.getBytes(StandardCharsets.UTF_8))); + script.setSize(content.getBytes(StandardCharsets.UTF_8).length); + return script; + } + + private static SkillAsset asset(String path, String content) { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + String hash = SkillHashes.sha256Hex(bytes); + SkillAsset asset = new SkillAsset(); + asset.setPath(path); + asset.setName("a.bin"); + asset.setMediaType("application/octet-stream"); + asset.setContentRef("sha256:" + hash); + asset.setContentHash(hash); + asset.setSize(bytes.length); + return asset; + } +} diff --git a/pom.xml b/pom.xml index 07e17d1..d9ed2c4 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,7 @@ easy-agents-embedding easy-agents-tool easy-agents-mcp + easy-agents-skill easy-agents-agent-runtime easy-agents-flow easy-agents-support @@ -430,6 +431,12 @@ ${revision} + + com.easyagents + easy-agents-skill + ${revision} + + com.easyagents easy-agents-agent-runtime From 848197b5562e546d8b9c5b378320453f5c064296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 8 Jun 2026 17:22:48 +0800 Subject: [PATCH 02/33] =?UTF-8?q?chore:=20=E8=B0=83=E6=95=B4=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E5=88=86=E6=94=AF=E4=B8=BA=E8=AF=95=E9=AA=8C=E6=80=A7?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d9ed2c4..446d648 100644 --- a/pom.xml +++ b/pom.xml @@ -34,8 +34,7 @@ - - 1.0.0 + 1.1.0-RC 17 1.3.0 UTF-8 From 66da0c9039b91d519ac80a76ef9e3f19e148e021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 17 Jul 2026 19:43:55 +0800 Subject: [PATCH 03/33] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BD=93=E6=A8=A1=E5=9E=8B=20HTTP=20=E4=BC=A0?= =?UTF-8?q?=E8=BE=93=E5=85=BC=E5=AE=B9=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 按模型地址自动选择 HTTP/1.1 或 HTTP/2 优先策略 - 复用并托管 AgentScope HTTP Transport 生命周期 - 补充协议解析、复用与 Provider 边界测试 --- .../AgentHttpTransportProvider.java | 132 ++++++++++++++++++ .../agentscope/AgentScopeModelFactory.java | 57 +++++++- .../runtime/model/AgentHttpVersionPolicy.java | 16 +++ .../agent/runtime/model/AgentModelSpec.java | 57 ++++++++ .../AgentHttpTransportProviderTest.java | 106 ++++++++++++++ 5 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProvider.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentHttpVersionPolicy.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProviderTest.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProvider.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProvider.java new file mode 100644 index 0000000..89c8884 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProvider.java @@ -0,0 +1,132 @@ +package com.easyagents.agent.runtime.agentscope; + +import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; +import io.agentscope.core.model.transport.HttpTransport; +import io.agentscope.core.model.transport.HttpTransportConfig; +import io.agentscope.core.model.transport.HttpTransportFactory; +import io.agentscope.core.model.transport.HttpVersion; +import io.agentscope.core.model.transport.JdkHttpTransport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.net.http.HttpClient; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 按 HTTP 版本策略提供进程级共享的 AgentScope Transport。 + */ +public final class AgentHttpTransportProvider { + + private static final Logger LOG = LoggerFactory.getLogger(AgentHttpTransportProvider.class); + private static final AgentHttpTransportProvider SHARED = new AgentHttpTransportProvider(); + + private final Map transports = new ConcurrentHashMap<>(); + + /** + * 创建 Transport 提供器。 + */ + private AgentHttpTransportProvider() { + } + + /** + * 获取进程级共享提供器。 + * + * @return 共享提供器 + */ + public static AgentHttpTransportProvider shared() { + return SHARED; + } + + /** + * 获取指定策略与基础 URL 对应的共享 Transport。 + * + * @param policy HTTP 版本策略 + * @param baseUrl 最终生效的模型基础 URL + * @return 共享 Transport + */ + public HttpTransport getTransport(AgentHttpVersionPolicy policy, String baseUrl) { + AgentHttpVersionPolicy effectivePolicy = resolveEffectivePolicy(policy, baseUrl); + if (effectivePolicy == AgentHttpVersionPolicy.AUTO) { + return HttpTransportFactory.getDefault(); + } + return transports.computeIfAbsent(effectivePolicy, this::createTransport); + } + + /** + * 解析请求实际使用的 HTTP 策略。 + * + *

明文 HTTP 固定使用 HTTP/1.1,避免 JDK 客户端发起 h2c Upgrade;HTTPS + * 保持 HTTP/2 优先并允许底层通过 ALPN 回退。显式策略始终覆盖 URL 判断。

+ * + * @param policy 配置的 HTTP 版本策略 + * @param baseUrl 最终生效的模型基础 URL + * @return 实际生效策略;无法识别 URL 时返回 AUTO + */ + public static AgentHttpVersionPolicy resolveEffectivePolicy(AgentHttpVersionPolicy policy, String baseUrl) { + AgentHttpVersionPolicy safePolicy = policy == null ? AgentHttpVersionPolicy.AUTO : policy; + if (safePolicy != AgentHttpVersionPolicy.AUTO) { + return safePolicy; + } + if (baseUrl == null || baseUrl.isBlank()) { + LOG.warn("Agent HTTP AUTO policy cannot infer protocol because base URL is missing; fallback to default transport"); + return AgentHttpVersionPolicy.AUTO; + } + try { + String scheme = URI.create(baseUrl.trim()).getScheme(); + if (scheme == null) { + return AgentHttpVersionPolicy.AUTO; + } + String normalizedScheme = scheme.toLowerCase(Locale.ROOT); + if ("http".equals(normalizedScheme)) { + return AgentHttpVersionPolicy.HTTP_1_1; + } + if ("https".equals(normalizedScheme)) { + return AgentHttpVersionPolicy.HTTP_2_PREFERRED; + } + LOG.warn("Agent HTTP AUTO policy does not support URL scheme '{}'; fallback to default transport", + normalizedScheme); + return AgentHttpVersionPolicy.AUTO; + } catch (IllegalArgumentException exception) { + LOG.warn("Agent HTTP AUTO policy cannot parse base URL; fallback to default transport"); + return AgentHttpVersionPolicy.AUTO; + } + } + + /** + * 创建并注册受 AgentScope 生命周期管理的 JDK Transport。 + * + * @param policy HTTP 版本策略 + * @return 新 Transport + */ + private HttpTransport createTransport(AgentHttpVersionPolicy policy) { + HttpVersion httpVersion = resolveHttpVersion(policy); + HttpTransportConfig config = HttpTransportConfig.builder() + .httpVersion(httpVersion) + .build(); + HttpClient httpClient = HttpClient.newBuilder() + .version(httpVersion.toJdkHttpVersion()) + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(config.getConnectTimeout()) + .build(); + HttpTransport transport = new JdkHttpTransport(httpClient, config); + // 注册后由 AgentScope JVM shutdown hook 统一关闭,避免每次 Agent 运行创建连接池。 + HttpTransportFactory.register(transport); + return transport; + } + + /** + * 将中立策略映射为 AgentScope HTTP 版本。 + * + * @param policy HTTP 版本策略 + * @return AgentScope HTTP 版本 + */ + static HttpVersion resolveHttpVersion(AgentHttpVersionPolicy policy) { + if (policy == AgentHttpVersionPolicy.HTTP_1_1) { + return HttpVersion.HTTP_1_1; + } + return HttpVersion.HTTP_2; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java index d408b31..15437d0 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java @@ -2,6 +2,7 @@ package com.easyagents.agent.runtime.agentscope; import com.easyagents.agent.runtime.AgentRuntimeException; import com.easyagents.agent.runtime.model.AgentGenerationOptions; +import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; import com.easyagents.agent.runtime.model.AgentModelFactory; import com.easyagents.agent.runtime.model.AgentModelProviderType; import com.easyagents.agent.runtime.model.AgentModelSpec; @@ -24,6 +25,24 @@ public class AgentScopeModelFactory implements AgentModelFactory { private static final String ARK_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; private static final String SILICONFLOW_BASE_URL = "https://api.siliconflow.cn/v1"; + private final AgentHttpTransportProvider httpTransportProvider; + + /** + * 使用进程级共享 Transport 提供器创建模型工厂。 + */ + public AgentScopeModelFactory() { + this(AgentHttpTransportProvider.shared()); + } + + /** + * 使用指定 Transport 提供器创建模型工厂。 + * + * @param httpTransportProvider Transport 提供器 + */ + AgentScopeModelFactory(AgentHttpTransportProvider httpTransportProvider) { + this.httpTransportProvider = httpTransportProvider; + } + @Override public Model create(AgentModelSpec modelSpec, AgentGenerationOptions generationOptions) { if (modelSpec == null) { @@ -31,6 +50,7 @@ public class AgentScopeModelFactory implements AgentModelFactory { } GenerateOptions options = toGenerateOptions(modelSpec, generationOptions); AgentModelProviderType providerType = modelSpec.getProviderType(); + validateHttpTransportSupport(providerType, modelSpec.getHttpVersionPolicy()); if (providerType == AgentModelProviderType.OLLAMA) { return buildOllama(modelSpec, options); } @@ -124,12 +144,14 @@ public class AgentScopeModelFactory implements AgentModelFactory { * @return 模型 */ private Model buildOpenAiCompatible(AgentModelSpec modelSpec, GenerateOptions options, String defaultBaseUrl) { + String baseUrl = resolveBaseUrl(modelSpec, defaultBaseUrl); OpenAIChatModel.Builder builder = OpenAIChatModel.builder() .apiKey(modelSpec.getApiKey()) .modelName(modelSpec.getModelName()) - .baseUrl(resolveBaseUrl(modelSpec, defaultBaseUrl)) + .baseUrl(baseUrl) .endpointPath(modelSpec.getEndpointPath()) .stream(Boolean.TRUE.equals(options.getStream())) + .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) .generateOptions(options); return builder.build(); } @@ -176,12 +198,14 @@ public class AgentScopeModelFactory implements AgentModelFactory { * @return 模型 */ private Model buildDeepSeek(AgentModelSpec modelSpec, GenerateOptions options) { + String baseUrl = resolveBaseUrl(modelSpec, DEEPSEEK_BASE_URL); OpenAIChatModel.Builder builder = OpenAIChatModel.builder() .apiKey(modelSpec.getApiKey()) .modelName(modelSpec.getModelName()) - .baseUrl(resolveBaseUrl(modelSpec, DEEPSEEK_BASE_URL)) + .baseUrl(baseUrl) .endpointPath(modelSpec.getEndpointPath()) .stream(Boolean.TRUE.equals(options.getStream())) + .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) .formatter(new DeepSeekFormatter()) .generateOptions(options); return builder.build(); @@ -195,12 +219,14 @@ public class AgentScopeModelFactory implements AgentModelFactory { * @return 模型 */ private Model buildGlm(AgentModelSpec modelSpec, GenerateOptions options) { + String baseUrl = resolveBaseUrl(modelSpec, GLM_BASE_URL); OpenAIChatModel.Builder builder = OpenAIChatModel.builder() .apiKey(modelSpec.getApiKey()) .modelName(modelSpec.getModelName()) - .baseUrl(resolveBaseUrl(modelSpec, GLM_BASE_URL)) + .baseUrl(baseUrl) .endpointPath(modelSpec.getEndpointPath()) .stream(Boolean.TRUE.equals(options.getStream())) + .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) .formatter(new GLMFormatter()) .generateOptions(options); return builder.build(); @@ -231,16 +257,39 @@ public class AgentScopeModelFactory implements AgentModelFactory { */ private Model buildDashScope(AgentModelSpec modelSpec, AgentGenerationOptions generationOptions, GenerateOptions options) { Boolean thinkingEnabled = generationOptions == null ? null : generationOptions.getThinkingEnabled(); + String baseUrl = modelSpec.getBaseUrl(); return DashScopeChatModel.builder() .apiKey(modelSpec.getApiKey()) .modelName(modelSpec.getModelName()) - .baseUrl(modelSpec.getBaseUrl()) + .baseUrl(baseUrl) .stream(Boolean.TRUE.equals(options.getStream())) .enableThinking(thinkingEnabled) + .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) .defaultOptions(options) .build(); } + /** + * 校验当前 Provider 是否支持显式 Agent HTTP Transport。 + * + * @param providerType 模型供应商类型 + * @param policy HTTP 版本策略 + */ + private void validateHttpTransportSupport(AgentModelProviderType providerType, + AgentHttpVersionPolicy policy) { + if (policy == null || policy == AgentHttpVersionPolicy.AUTO) { + return; + } + if (providerType == AgentModelProviderType.ANTHROPIC + || providerType == AgentModelProviderType.GEMINI + || providerType == AgentModelProviderType.OLLAMA) { + throw new AgentRuntimeException( + "Agent HTTP transport policy " + policy + + " is not supported by provider " + providerType + + "; use AUTO for this provider."); + } + } + /** * 优先使用调用方传入的基础 URL,未传入时使用供应商默认地址。 * diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentHttpVersionPolicy.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentHttpVersionPolicy.java new file mode 100644 index 0000000..583aca6 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentHttpVersionPolicy.java @@ -0,0 +1,16 @@ +package com.easyagents.agent.runtime.model; + +/** + * Agent 模型调用使用的 HTTP 版本策略。 + */ +public enum AgentHttpVersionPolicy { + + /** 按基础 URL 协议自动选择:HTTP 使用 1.1,HTTPS 优先使用 2。 */ + AUTO, + + /** 强制使用 HTTP/1.1。 */ + HTTP_1_1, + + /** 优先使用 HTTP/2,并允许 JDK 客户端按协议能力回退。 */ + HTTP_2_PREFERRED +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java index e61ee99..59bc84a 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java @@ -13,6 +13,9 @@ public class AgentModelSpec { private String baseUrl; private String endpointPath; private String apiKey; + private boolean supportImage; + private boolean supportImageBase64Only; + private AgentHttpVersionPolicy httpVersionPolicy = AgentHttpVersionPolicy.AUTO; private Map metadata = new LinkedHashMap<>(); /** @@ -105,6 +108,60 @@ public class AgentModelSpec { this.apiKey = apiKey; } + /** + * 判断模型是否支持图片输入。 + * + * @return 支持图片时返回 true + */ + public boolean isSupportImage() { + return supportImage; + } + + /** + * 设置模型是否支持图片输入。 + * + * @param supportImage 是否支持图片 + */ + public void setSupportImage(boolean supportImage) { + this.supportImage = supportImage; + } + + /** + * 判断模型是否只接受 Base64 图片。 + * + * @return 仅接受 Base64 时返回 true + */ + public boolean isSupportImageBase64Only() { + return supportImageBase64Only; + } + + /** + * 设置模型是否只接受 Base64 图片。 + * + * @param supportImageBase64Only 是否只接受 Base64 图片 + */ + public void setSupportImageBase64Only(boolean supportImageBase64Only) { + this.supportImageBase64Only = supportImageBase64Only; + } + + /** + * 获取 Agent 模型调用的 HTTP 版本策略。 + * + * @return HTTP 版本策略 + */ + public AgentHttpVersionPolicy getHttpVersionPolicy() { + return httpVersionPolicy; + } + + /** + * 设置 Agent 模型调用的 HTTP 版本策略。 + * + * @param httpVersionPolicy HTTP 版本策略 + */ + public void setHttpVersionPolicy(AgentHttpVersionPolicy httpVersionPolicy) { + this.httpVersionPolicy = httpVersionPolicy == null ? AgentHttpVersionPolicy.AUTO : httpVersionPolicy; + } + /** * 获取元数据。 * diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProviderTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProviderTest.java new file mode 100644 index 0000000..a980ea7 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentHttpTransportProviderTest.java @@ -0,0 +1,106 @@ +package com.easyagents.agent.runtime.agentscope; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import com.easyagents.agent.runtime.model.AgentGenerationOptions; +import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; +import com.easyagents.agent.runtime.model.AgentModelProviderType; +import com.easyagents.agent.runtime.model.AgentModelSpec; +import io.agentscope.core.model.transport.HttpTransport; +import io.agentscope.core.model.transport.HttpTransportFactory; +import io.agentscope.core.model.transport.HttpVersion; +import io.agentscope.core.model.transport.JdkHttpTransport; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.net.http.HttpClient; + +/** + * Agent HTTP Transport 策略测试。 + */ +public class AgentHttpTransportProviderTest { + + /** + * 验证相同策略复用 Transport,且显式策略使用预期 JDK HTTP 版本。 + * + * @throws Exception 反射读取 JDK client 失败时抛出 + */ + @Test + public void shouldReuseTransportAndConfigureExpectedHttpVersion() throws Exception { + AgentHttpTransportProvider provider = AgentHttpTransportProvider.shared(); + + HttpTransport http11 = provider.getTransport(AgentHttpVersionPolicy.HTTP_1_1, "https://example.com"); + HttpTransport http11Again = provider.getTransport(AgentHttpVersionPolicy.HTTP_1_1, "http://example.com"); + HttpTransport http2 = provider.getTransport(AgentHttpVersionPolicy.HTTP_2_PREFERRED, "http://example.com"); + + Assert.assertSame(http11, http11Again); + Assert.assertNotSame(http11, http2); + Assert.assertEquals(HttpClient.Version.HTTP_1_1, httpClient(http11).version()); + Assert.assertEquals(HttpClient.Version.HTTP_2, httpClient(http2).version()); + Assert.assertTrue(HttpTransportFactory.isManaged(http11)); + Assert.assertTrue(HttpTransportFactory.isManaged(http2)); + Assert.assertEquals(HttpVersion.HTTP_1_1, + AgentHttpTransportProvider.resolveHttpVersion(AgentHttpVersionPolicy.HTTP_1_1)); + Assert.assertEquals(HttpVersion.HTTP_2, + AgentHttpTransportProvider.resolveHttpVersion(AgentHttpVersionPolicy.HTTP_2_PREFERRED)); + } + + /** + * 验证 AUTO 根据 URL 协议选择安全 Transport。 + * + * @throws Exception 反射读取 JDK client 失败时抛出 + */ + @Test + public void autoShouldResolveTransportFromBaseUrlScheme() throws Exception { + AgentHttpTransportProvider provider = AgentHttpTransportProvider.shared(); + HttpTransport http = provider.getTransport(AgentHttpVersionPolicy.AUTO, "http://example.com/v1"); + HttpTransport https = provider.getTransport(AgentHttpVersionPolicy.AUTO, "https://example.com/v1"); + HttpTransport unknown = provider.getTransport(AgentHttpVersionPolicy.AUTO, "example.com/v1"); + + Assert.assertSame(provider.getTransport(AgentHttpVersionPolicy.HTTP_1_1, null), http); + Assert.assertSame(provider.getTransport(AgentHttpVersionPolicy.HTTP_2_PREFERRED, null), https); + Assert.assertEquals(HttpClient.Version.HTTP_1_1, httpClient(http).version()); + Assert.assertEquals(HttpClient.Version.HTTP_2, httpClient(https).version()); + Assert.assertSame(HttpTransportFactory.getDefault(), unknown); + Assert.assertEquals(AgentHttpVersionPolicy.HTTP_1_1, + AgentHttpTransportProvider.resolveEffectivePolicy(AgentHttpVersionPolicy.AUTO, "HTTP://EXAMPLE.COM")); + Assert.assertEquals(AgentHttpVersionPolicy.HTTP_2_PREFERRED, + AgentHttpTransportProvider.resolveEffectivePolicy(AgentHttpVersionPolicy.AUTO, "HTTPS://EXAMPLE.COM")); + Assert.assertEquals(AgentHttpVersionPolicy.AUTO, + AgentHttpTransportProvider.resolveEffectivePolicy(AgentHttpVersionPolicy.AUTO, "not a uri")); + } + + /** + * 验证不支持注入 Transport 的 Provider 会返回明确错误。 + */ + @Test + public void unsupportedProviderShouldRejectExplicitHttpPolicy() { + AgentModelSpec spec = new AgentModelSpec(); + spec.setProviderType(AgentModelProviderType.ANTHROPIC); + spec.setModelName("claude-test"); + spec.setApiKey("test-key"); + spec.setHttpVersionPolicy(AgentHttpVersionPolicy.HTTP_1_1); + + try { + new AgentScopeModelFactory().create(spec, new AgentGenerationOptions()); + Assert.fail("Expected unsupported HTTP transport policy error"); + } catch (AgentRuntimeException exception) { + Assert.assertTrue(exception.getMessage().contains("HTTP_1_1")); + Assert.assertTrue(exception.getMessage().contains("ANTHROPIC")); + } + } + + /** + * 获取 Transport 内部复用的 JDK HttpClient。 + * + * @param transport Transport 实例 + * @return JDK HttpClient + * @throws Exception 反射失败时抛出 + */ + private HttpClient httpClient(HttpTransport transport) throws Exception { + Assert.assertTrue(transport instanceof JdkHttpTransport); + Field field = JdkHttpTransport.class.getDeclaredField("client"); + field.setAccessible(true); + return (HttpClient) field.get(transport); + } +} From f057900f7a7472e4b3ebbd109c172294e4c0c8ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 17 Jul 2026 19:45:37 +0800 Subject: [PATCH 04/33] =?UTF-8?q?feat:=20=E6=89=93=E9=80=9A=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BD=93=E5=9B=BE=E7=89=87=E5=AA=92=E4=BD=93=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 增加稳定媒体引用解析与模型图片能力校验 - 修复图片 Data URI 与模型完整文本读取 - 补充媒体解析和消息兼容测试 --- .../agent/runtime/AgentInitRequest.java | 24 +++ .../agentscope/AgentScopeMessageAdapter.java | 35 +++- .../agentscope/AgentScopeReActRuntime.java | 8 + .../MediaReferenceInterceptor.java | 159 ++++++++++++++++++ .../runtime/media/AgentMediaResolver.java | 17 ++ .../runtime/media/AgentMediaResource.java | 29 ++++ .../runtime/message/AgentMediaBlock.java | 20 +++ .../AgentScopeStatefulRuntimeTest.java | 15 ++ .../MediaReferenceInterceptorTest.java | 117 +++++++++++++ .../core/agent/react/ReActAgent.java | 2 +- .../easyagents/core/message/AiMessage.java | 19 ++- .../easyagents/core/model/chat/ChatModel.java | 2 +- .../chat/ChatObservabilityInterceptor.java | 2 +- .../parser/impl/DefaultAiMessageParser.java | 12 +- .../com/easyagents/core/util/ImageUtil.java | 4 +- .../core/message/AiMessageTest.java | 51 ++++++ .../parser/DefaultAiMessageParserTest.java | 87 ++++++++++ .../core/test/util/ImageUtilTest.java | 36 ++++ .../flow/support/provider/EasyAgentsLlm.java | 2 +- 19 files changed, 629 insertions(+), 12 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java create mode 100644 easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java create mode 100644 easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java create mode 100644 easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java index ac2c7aa..1e647b8 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentInitRequest.java @@ -1,6 +1,7 @@ package com.easyagents.agent.runtime; import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetriever; +import com.easyagents.agent.runtime.media.AgentMediaResolver; import com.easyagents.agent.runtime.persistence.conversation.AgentConversationRecorder; import com.easyagents.agent.runtime.persistence.conversation.noop.NoopAgentConversationRecorder; import com.easyagents.agent.runtime.persistence.session.AgentSessionStore; @@ -55,6 +56,11 @@ public class AgentInitRequest { */ private Map metadata = new LinkedHashMap<>(); + /** + * 媒体引用解析器,仅在模型调用前解析稳定引用。 + */ + private AgentMediaResolver mediaResolver; + /** * 获取会话ID。 * @@ -200,4 +206,22 @@ public class AgentInitRequest { public void setMetadata(Map metadata) { this.metadata = metadata == null ? new LinkedHashMap<>() : metadata; } + + /** + * 获取媒体引用解析器。 + * + * @return 媒体引用解析器 + */ + public AgentMediaResolver getMediaResolver() { + return mediaResolver; + } + + /** + * 设置媒体引用解析器。 + * + * @param mediaResolver 媒体引用解析器 + */ + public void setMediaResolver(AgentMediaResolver mediaResolver) { + this.mediaResolver = mediaResolver; + } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java index 71672eb..6b55439 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeMessageAdapter.java @@ -5,7 +5,9 @@ import io.agentscope.core.message.*; import java.time.Instant; import java.time.OffsetDateTime; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -15,6 +17,9 @@ import java.util.Map; */ public class AgentScopeMessageAdapter { + /** AgentScope URLSource 中用于持久化业务媒体引用的私有协议。 */ + public static final String MEDIA_REFERENCE_SCHEME = "easyagents-media://"; + /** * 将运行时消息转换为 AgentScope 消息。 * @@ -276,6 +281,11 @@ public class AgentScopeMessageAdapter { .data(block.getData()) .build(); } + if (block.getReference() != null && !block.getReference().isBlank()) { + String encodedReference = Base64.getUrlEncoder().withoutPadding() + .encodeToString(block.getReference().getBytes(StandardCharsets.UTF_8)); + return URLSource.builder().url(MEDIA_REFERENCE_SCHEME + encodedReference).build(); + } return URLSource.builder().url(block.getUrl()).build(); } @@ -286,7 +296,30 @@ public class AgentScopeMessageAdapter { return; } if (source instanceof URLSource urlSource) { - block.setUrl(urlSource.getUrl()); + String url = urlSource.getUrl(); + if (url != null && url.startsWith(MEDIA_REFERENCE_SCHEME)) { + block.setReference(decodeReference(url)); + } else { + block.setUrl(url); + } + } + } + + /** + * 从 AgentScope 私有 URLSource 解码业务媒体引用。 + * + * @param url 私有协议 URL + * @return 原始业务媒体引用 + */ + public static String decodeReference(String url) { + if (url == null || !url.startsWith(MEDIA_REFERENCE_SCHEME)) { + return null; + } + String encoded = url.substring(MEDIA_REFERENCE_SCHEME.length()); + try { + return new String(Base64.getUrlDecoder().decode(encoded), StandardCharsets.UTF_8); + } catch (IllegalArgumentException error) { + throw new IllegalArgumentException("Invalid agent media reference.", error); } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java index 27e073d..7530e33 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java @@ -3,6 +3,7 @@ package com.easyagents.agent.runtime.agentscope; import com.easyagents.agent.runtime.*; import com.easyagents.agent.runtime.event.*; import com.easyagents.agent.runtime.event.interceptor.AutoContextInterceptor; +import com.easyagents.agent.runtime.event.interceptor.MediaReferenceInterceptor; import com.easyagents.agent.runtime.event.interceptor.ToolHitlInterceptor; import com.easyagents.agent.runtime.event.observer.AgentRuntimeErrorObserver; import com.easyagents.agent.runtime.event.observer.ReasoningLifecycleObserver; @@ -303,6 +304,12 @@ public class AgentScopeReActRuntime implements AgentRuntime { if (userMessage.getContentBlocks() == null || userMessage.getContentBlocks().isEmpty()) { throw new AgentRuntimeException("Agent user message content is required."); } + boolean containsImage = userMessage.getContentBlocks().stream() + .anyMatch(block -> block instanceof AgentMediaBlock mediaBlock + && "image".equalsIgnoreCase(mediaBlock.getMediaKind())); + if (containsImage && !initRequest.getAgentDefinition().getModelSpec().isSupportImage()) { + throw new AgentRuntimeException("The configured model does not support image input."); + } } /** @@ -1087,6 +1094,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { if (memory instanceof AutoContextMemory) { interceptors.add(new AutoContextInterceptor(eventBridge, memoryResult.getAutoContextConfig())); } + interceptors.add(new MediaReferenceInterceptor(initRequest.getMediaResolver())); List runtimeToolSpecs = mergeToolSpecs(definition.getToolSpecs(), toolkitBuildResult.mcpToolSpecs(), toolkitBuildResult.operateToolSpecs()); interceptors.add(new ToolHitlInterceptor(eventBridge, approvalCoordinator, diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java new file mode 100644 index 0000000..ec99472 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptor.java @@ -0,0 +1,159 @@ +package com.easyagents.agent.runtime.event.interceptor; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter; +import com.easyagents.agent.runtime.event.AgentRuntimeInterceptor; +import com.easyagents.agent.runtime.media.AgentMediaResolver; +import com.easyagents.agent.runtime.media.AgentMediaResource; +import io.agentscope.core.hook.HookEvent; +import io.agentscope.core.hook.PreReasoningEvent; +import io.agentscope.core.message.*; +import reactor.core.publisher.Mono; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * 在推理前将持久化的业务媒体引用解析为模型可用的 Base64 内容。 + * + *

输入消息会被深拷贝,AgentScope memory/session 中仍保留小体积稳定引用。

+ */ +public class MediaReferenceInterceptor implements AgentRuntimeInterceptor { + + private final AgentMediaResolver mediaResolver; + + /** + * 创建媒体引用干预器。 + * + * @param mediaResolver 媒体引用解析器 + */ + public MediaReferenceInterceptor(AgentMediaResolver mediaResolver) { + this.mediaResolver = mediaResolver; + } + + /** + * 解析推理输入中的内部媒体引用。 + * + * @param event AgentScope Hook 事件 + * @param Hook 事件类型 + * @return 处理后的事件 + */ + @Override + public Mono intercept(T event) { + if (event instanceof PreReasoningEvent preReasoningEvent) { + preReasoningEvent.setInputMessages(resolveMessages(preReasoningEvent.getInputMessages())); + } + return Mono.just(event); + } + + /** + * 在 AutoContext 完成消息重写后执行媒体解析。 + * + * @return 执行优先级 + */ + @Override + public int priority() { + return 10; + } + + private List resolveMessages(List messages) { + if (messages == null || messages.isEmpty()) { + return List.of(); + } + List resolved = new ArrayList<>(messages.size()); + for (Msg message : messages) { + List content = resolveBlocks(message.getContent()); + resolved.add(Msg.builder() + .id(message.getId()) + .name(message.getName()) + .role(message.getRole()) + .content(content) + .metadata(message.getMetadata()) + .timestamp(message.getTimestamp()) + .build()); + } + return resolved; + } + + private List resolveBlocks(List blocks) { + if (blocks == null || blocks.isEmpty()) { + return List.of(); + } + List resolved = new ArrayList<>(blocks.size()); + for (ContentBlock block : blocks) { + if (block instanceof ImageBlock imageBlock) { + resolved.add(resolveImage(imageBlock)); + } else if (block instanceof AudioBlock audioBlock) { + resolved.add(resolveAudio(audioBlock)); + } else if (block instanceof VideoBlock videoBlock) { + resolved.add(resolveVideo(videoBlock)); + } else { + resolved.add(block); + } + } + return resolved; + } + + private ImageBlock resolveImage(ImageBlock block) { + AgentMediaResource resource = resolve(block.getSource()); + if (resource == null) { + return block; + } + return ImageBlock.builder() + .source(base64Source(resource)) + .minPixels(block.getMinPixels()) + .maxPixels(block.getMaxPixels()) + .build(); + } + + private AudioBlock resolveAudio(AudioBlock block) { + AgentMediaResource resource = resolve(block.getSource()); + return resource == null ? block : AudioBlock.builder().source(base64Source(resource)).build(); + } + + private VideoBlock resolveVideo(VideoBlock block) { + AgentMediaResource resource = resolve(block.getSource()); + if (resource == null) { + return block; + } + return VideoBlock.builder() + .source(base64Source(resource)) + .fps(block.getFps()) + .maxFrames(block.getMaxFrames()) + .minPixels(block.getMinPixels()) + .maxPixels(block.getMaxPixels()) + .totalPixels(block.getTotalPixels()) + .build(); + } + + private AgentMediaResource resolve(Source source) { + if (!(source instanceof URLSource urlSource) + || urlSource.getUrl() == null + || !urlSource.getUrl().startsWith(AgentScopeMessageAdapter.MEDIA_REFERENCE_SCHEME)) { + return null; + } + if (mediaResolver == null) { + throw new AgentRuntimeException("Agent media resolver is required for media references."); + } + String reference; + try { + reference = AgentScopeMessageAdapter.decodeReference(urlSource.getUrl()); + } catch (IllegalArgumentException error) { + throw new AgentRuntimeException("Agent media reference is invalid.", error); + } + AgentMediaResource resource = mediaResolver.resolve(reference); + if (resource == null || resource.bytes().length == 0 || resource.mimeType() == null + || resource.mimeType().isBlank()) { + throw new AgentRuntimeException("Agent media resource is empty."); + } + return resource; + } + + private Base64Source base64Source(AgentMediaResource resource) { + return Base64Source.builder() + .mediaType(resource.mimeType()) + .data(Base64.getEncoder().encodeToString(resource.bytes())) + .build(); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java new file mode 100644 index 0000000..0b6d424 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResolver.java @@ -0,0 +1,17 @@ +package com.easyagents.agent.runtime.media; + +/** + * 在模型调用边界解析业务侧稳定媒体引用。 + */ +@FunctionalInterface +public interface AgentMediaResolver { + + /** + * 解析媒体引用。 + * + * @param reference 业务侧稳定媒体引用 + * @return 媒体资源 + * @throws RuntimeException 引用无效、越权或资源读取失败时抛出 + */ + AgentMediaResource resolve(String reference); +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java new file mode 100644 index 0000000..f229a40 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/media/AgentMediaResource.java @@ -0,0 +1,29 @@ +package com.easyagents.agent.runtime.media; + +import java.util.Arrays; + +/** + * 模型调用前解析得到的媒体资源。 + * + * @param mimeType MIME 类型 + * @param bytes 媒体字节 + */ +public record AgentMediaResource(String mimeType, byte[] bytes) { + + /** + * 创建不可变媒体资源。 + */ + public AgentMediaResource { + bytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); + } + + /** + * 返回媒体字节副本。 + * + * @return 媒体字节副本 + */ + @Override + public byte[] bytes() { + return Arrays.copyOf(bytes, bytes.length); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java index 8835e51..3ee561a 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/message/AgentMediaBlock.java @@ -9,6 +9,7 @@ import java.util.Map; public class AgentMediaBlock extends AgentContentBlock { private String mimeType; + private String reference; private String url; private String data; private Integer minPixels; @@ -64,6 +65,24 @@ public class AgentMediaBlock extends AgentContentBlock { this.mimeType = mimeType; } + /** + * 获取由业务侧解析的稳定媒体引用。 + * + * @return 媒体引用 + */ + public String getReference() { + return reference; + } + + /** + * 设置由业务侧解析的稳定媒体引用。 + * + * @param reference 媒体引用 + */ + public void setReference(String reference) { + this.reference = reference; + } + /** * 获取 URL。 * @@ -209,6 +228,7 @@ public class AgentMediaBlock extends AgentContentBlock { Map metadata = new LinkedHashMap<>(getMetadata()); metadata.put("mediaKind", mediaKind); metadata.put("mimeType", mimeType); + metadata.put("reference", reference); metadata.put("url", url); metadata.put("data", data); metadata.put("minPixels", minPixels); diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java index ba15b66..50028ae 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java @@ -17,6 +17,7 @@ import com.easyagents.agent.runtime.memory.AgentMemoryPolicy; import com.easyagents.agent.runtime.memory.AgentMemorySnapshot; import com.easyagents.agent.runtime.message.AgentMessage; import com.easyagents.agent.runtime.message.AgentMessageRole; +import com.easyagents.agent.runtime.message.AgentMediaBlock; import com.easyagents.agent.runtime.model.AgentModelSpec; import com.easyagents.agent.runtime.persistence.session.memory.InMemoryAgentSessionStore; import com.easyagents.agent.runtime.skill.AgentSkillBoxSpec; @@ -78,6 +79,20 @@ public class AgentScopeStatefulRuntimeTest { runtime.stream(message); } + @Test(expected = AgentRuntimeException.class) + public void shouldRejectImageWhenModelCapabilityIsDisabled() { + AgentScopeReActRuntime runtime = fakeRuntime(); + AgentInitRequest request = initRequest(); + runtime.init(request); + AgentMessage message = new AgentMessage(); + message.setRole(AgentMessageRole.USER); + AgentMediaBlock image = new AgentMediaBlock("image"); + image.setReference("draft:upload-1"); + message.setContentBlocks(List.of(image)); + + runtime.stream(message); + } + @Test(expected = AgentRuntimeException.class) public void shouldRejectDuplicateInit() { AgentScopeReActRuntime runtime = fakeRuntime(); diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java new file mode 100644 index 0000000..6fa1cf2 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/event/interceptor/MediaReferenceInterceptorTest.java @@ -0,0 +1,117 @@ +package com.easyagents.agent.runtime.event.interceptor; + +import com.easyagents.agent.runtime.agentscope.AgentScopeMessageAdapter; +import com.easyagents.agent.runtime.media.AgentMediaResource; +import com.easyagents.agent.runtime.message.AgentMediaBlock; +import io.agentscope.core.ReActAgent; +import io.agentscope.core.hook.PreReasoningEvent; +import io.agentscope.core.message.Base64Source; +import io.agentscope.core.message.ImageBlock; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.message.URLSource; +import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.model.GenerateOptions; +import io.agentscope.core.model.Model; +import io.agentscope.core.model.ToolSchema; +import io.agentscope.core.tool.Toolkit; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * 测试模型调用边界的媒体引用解析。 + */ +public class MediaReferenceInterceptorTest { + + /** + * 验证私有引用仅在模型调用边界转换为 Base64,原始记忆消息保持稳定引用。 + */ + @Test + public void shouldResolveReferenceToBase64WithoutMutatingOriginalMessage() { + AgentScopeMessageAdapter adapter = new AgentScopeMessageAdapter(); + AgentMediaBlock mediaBlock = new AgentMediaBlock("image"); + mediaBlock.setReference("draft:upload-1"); + mediaBlock.setMimeType("image/png"); + Msg original = Msg.builder() + .id("message-1") + .name("user") + .role(MsgRole.USER) + .content(adapter.toContentBlock(mediaBlock)) + .build(); + AtomicReference resolvedReference = new AtomicReference<>(); + MediaReferenceInterceptor interceptor = new MediaReferenceInterceptor(reference -> { + resolvedReference.set(reference); + return new AgentMediaResource("image/png", "png-data".getBytes(StandardCharsets.UTF_8)); + }); + ReActAgent agent = ReActAgent.builder() + .name("media-test-agent") + .sysPrompt("system") + .model(new EmptyModel()) + .toolkit(new Toolkit()) + .build(); + PreReasoningEvent event = new PreReasoningEvent(agent, "reasoning-1", null, List.of(original)); + + interceptor.intercept(event).block(); + + ImageBlock originalImage = original.getFirstContentBlock(ImageBlock.class); + ImageBlock resolvedImage = event.getInputMessages().get(0).getFirstContentBlock(ImageBlock.class); + Assert.assertTrue(originalImage.getSource() instanceof URLSource); + Assert.assertTrue(((URLSource) originalImage.getSource()).getUrl() + .startsWith(AgentScopeMessageAdapter.MEDIA_REFERENCE_SCHEME)); + Assert.assertTrue(resolvedImage.getSource() instanceof Base64Source); + Assert.assertEquals("draft:upload-1", resolvedReference.get()); + Assert.assertEquals("cG5nLWRhdGE=", ((Base64Source) resolvedImage.getSource()).getData()); + } + + /** + * 验证消息适配器序列化后仍能恢复不透明媒体引用。 + */ + @Test + public void shouldRoundTripStableReferenceThroughMessageAdapter() { + AgentScopeMessageAdapter adapter = new AgentScopeMessageAdapter(); + AgentMediaBlock mediaBlock = new AgentMediaBlock("image"); + mediaBlock.setReference("formal:message-1:0:image/png"); + + AgentMediaBlock restored = (AgentMediaBlock) adapter.toAgentBlock(adapter.toContentBlock(mediaBlock)); + + Assert.assertEquals("formal:message-1:0:image/png", restored.getReference()); + Assert.assertNull(restored.getData()); + Assert.assertNull(restored.getUrl()); + } + + /** + * 仅用于构建真实 AgentScope 事件的空模型。 + */ + private static final class EmptyModel implements Model { + + /** + * 返回空响应流,当前测试不会实际调用模型。 + * + * @param messages 输入消息 + * @param toolSchemas 工具定义 + * @param options 生成参数 + * @return 空响应流 + */ + @Override + public Flux stream(List messages, + List toolSchemas, + GenerateOptions options) { + return Flux.empty(); + } + + /** + * 返回测试模型名称。 + * + * @return 测试模型名称 + */ + @Override + public String getModelName() { + return "media-test-model"; + } + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java b/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java index 2e89959..bfcbcdb 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/agent/react/ReActAgent.java @@ -233,7 +233,7 @@ public class ReActAgent implements IAgent { AiMessageResponse response = chatModel.chat(memoryPrompt, chatOptions); notifyOnChatResponse(response); - String content = response.getMessage().getContent(); + String content = response.getMessage().getTextContent(); AiMessage message = new AiMessage(content); // 请求用户输入 diff --git a/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java b/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java index 7daa0b7..d986eb2 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/message/AiMessage.java @@ -19,6 +19,9 @@ import com.easyagents.core.util.StringUtil; import java.util.*; +/** + * 表示模型生成的文本、推理内容和工具调用消息。 + */ public class AiMessage extends AbstractTextMessage { private Integer index; @@ -192,8 +195,13 @@ public class AiMessage extends AbstractTextMessage { this.localTotalTokens = localTotalTokens; } + /** + * 获取当前消息的完整回复文本。 + * + * @return 已聚合的完整文本;未聚合时返回当前文本 + */ public String getFullContent() { - return fullContent; + return fullContent != null ? fullContent : content; } public void setFullContent(String fullContent) { @@ -226,7 +234,7 @@ public class AiMessage extends AbstractTextMessage { @Override public String getTextContent() { - return fullContent; + return getFullContent(); } /** @@ -283,8 +291,13 @@ public class AiMessage extends AbstractTextMessage { this.toolCalls = toolCalls; } + /** + * 获取当前消息的完整推理文本。 + * + * @return 已聚合的完整推理文本;未聚合时返回当前推理文本 + */ public String getFullReasoningContent() { - return fullReasoningContent; + return fullReasoningContent != null ? fullReasoningContent : reasoningContent; } public void setFullReasoningContent(String fullReasoningContent) { diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java index c2a0c2d..b9e9225 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatModel.java @@ -33,7 +33,7 @@ public interface ChatModel { if (response != null && response.isError()) { throw new ModelException(response.getErrorMessage()); } - return response != null && response.getMessage() != null ? response.getMessage().getContent() : null; + return response != null && response.getMessage() != null ? response.getMessage().getTextContent() : null; } default AiMessageResponse chat(Prompt prompt) { diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java index d7274fa..7f6ac9c 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatObservabilityInterceptor.java @@ -177,7 +177,7 @@ public class ChatObservabilityInterceptor implements ChatInterceptor { private void enrichSpan(Span span, AiMessage msg) { if (msg != null) { span.setAttribute("llm.total_tokens", msg.getEffectiveTotalTokens()); - String content = msg.getContent(); + String content = msg.getTextContent(); if (content != null) { span.setAttribute("llm.response", content.substring(0, Math.min(content.length(), MAX_RESPONSE_LENGTH_FOR_SPAN))); diff --git a/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java b/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java index fd31027..578f6bd 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/parser/impl/DefaultAiMessageParser.java @@ -32,6 +32,9 @@ import java.util.List; import java.util.Map; +/** + * 解析 OpenAI-compatible 同步响应和流式增量消息。 + */ public class DefaultAiMessageParser implements AiMessageParser { private JSONPath contentPath; @@ -171,11 +174,16 @@ public class DefaultAiMessageParser implements AiMessageParser { } } else { if (this.contentPath != null) { - aiMessage.setContent((String) this.contentPath.eval(rootJson)); + String content = (String) this.contentPath.eval(rootJson); + // 非流式响应已经是完整结果,同时填充当前内容和完整内容。 + aiMessage.setContent(content); + aiMessage.setFullContent(content); } if (this.reasoningContentPath != null) { - aiMessage.setReasoningContent((String) this.reasoningContentPath.eval(rootJson)); + String reasoningContent = (String) this.reasoningContentPath.eval(rootJson); + aiMessage.setReasoningContent(reasoningContent); + aiMessage.setFullReasoningContent(reasoningContent); } if (this.toolCallsJsonPath != null) { toolCallsJsonArray = (JSONArray) this.toolCallsJsonPath.eval(rootJson); diff --git a/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java b/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java index 97a6819..f3fa7a6 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/util/ImageUtil.java @@ -50,7 +50,7 @@ public class ImageUtil { } /** - * 将图片 URL 转换为 Data URI 格式的字符串(例如:image/jpeg;base64,...) + * 将图片 URL 转换为 Data URI 格式的字符串(例如:data:image/jpeg;base64,...) * * @throws IllegalArgumentException 如果 URL 无效或无法获取内容 */ @@ -108,6 +108,6 @@ public class ImageUtil { public static String imageBytesToDataUri(byte[] data, String mimeType) { String base64 = Base64.getEncoder().encodeToString(data); - return mimeType + ";base64," + base64; + return "data:" + mimeType + ";base64," + base64; } } diff --git a/easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java b/easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java new file mode 100644 index 0000000..e626a6c --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/message/AiMessageTest.java @@ -0,0 +1,51 @@ +package com.easyagents.core.message; + +import com.easyagents.core.util.LocalTokenCounter; +import org.junit.Assert; +import org.junit.Test; + +/** + * AI 消息文本语义测试。 + */ +public class AiMessageTest { + + /** + * 验证仅设置当前内容时,完整文本接口仍能返回模型回复。 + */ + @Test + public void currentContentShouldBeAvailableAsFullText() { + AiMessage message = new AiMessage(); + message.setContent("2026"); + + Assert.assertEquals("2026", message.getContent()); + Assert.assertEquals("2026", message.getFullContent()); + Assert.assertEquals("2026", message.getTextContent()); + Assert.assertTrue(LocalTokenCounter.countCompletionTokens(message) + > LocalTokenCounter.countCompletionTokens(new AiMessage())); + } + + /** + * 验证流式消息保留增量内容,同时完整文本接口返回聚合内容。 + */ + @Test + public void aggregatedContentShouldNotReplaceStreamingDelta() { + AiMessage message = new AiMessage(); + message.setContent("26"); + message.setFullContent("2026"); + + Assert.assertEquals("26", message.getContent()); + Assert.assertEquals("2026", message.getFullContent()); + Assert.assertEquals("2026", message.getTextContent()); + } + + /** + * 验证仅设置当前推理内容时,完整推理接口能够正确回退。 + */ + @Test + public void reasoningContentShouldBeAvailableAsFullReasoningText() { + AiMessage message = new AiMessage(); + message.setReasoningContent("识别图片中的数字"); + + Assert.assertEquals("识别图片中的数字", message.getFullReasoningContent()); + } +} diff --git a/easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java b/easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java new file mode 100644 index 0000000..52ed9bb --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/test/parser/DefaultAiMessageParserTest.java @@ -0,0 +1,87 @@ +package com.easyagents.core.test.parser; + +import com.alibaba.fastjson2.JSON; +import com.easyagents.core.message.AiMessage; +import com.easyagents.core.model.chat.ChatContext; +import com.easyagents.core.model.chat.ChatOptions; +import com.easyagents.core.parser.impl.DefaultAiMessageParser; +import org.junit.Assert; +import org.junit.Test; + +/** + * OpenAI-compatible AI 消息解析测试。 + */ +public class DefaultAiMessageParserTest { + + /** + * 验证非流式响应同时填充当前内容和完整内容。 + */ + @Test + public void nonStreamingResponseShouldPopulateCompleteContent() { + String response = """ + { + "choices": [{ + "message": { + "content": "2026", + "reasoning_content": "读取图片中的数字" + }, + "index": 0, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 221, + "completion_tokens": 51, + "total_tokens": 272 + } + } + """; + + AiMessage message = DefaultAiMessageParser.getOpenAIMessageParser() + .parse(JSON.parseObject(response), context(false)); + + Assert.assertEquals("2026", message.getContent()); + Assert.assertEquals("2026", message.getFullContent()); + Assert.assertEquals("2026", message.getTextContent()); + Assert.assertEquals("读取图片中的数字", message.getReasoningContent()); + Assert.assertEquals("读取图片中的数字", message.getFullReasoningContent()); + Assert.assertEquals(Integer.valueOf(272), message.getTotalTokens()); + } + + /** + * 验证流式解析继续暴露原始增量内容。 + */ + @Test + public void streamingResponseShouldKeepDeltaContent() { + String response = """ + { + "choices": [{ + "delta": { + "content": "20", + "reasoning_content": "读取" + }, + "index": 0 + }] + } + """; + + AiMessage message = DefaultAiMessageParser.getOpenAIMessageParser() + .parse(JSON.parseObject(response), context(true)); + + Assert.assertEquals("20", message.getContent()); + Assert.assertEquals("读取", message.getReasoningContent()); + } + + /** + * 创建指定流式模式的解析上下文。 + * + * @param streaming 是否解析流式增量 + * @return 解析上下文 + */ + private ChatContext context(boolean streaming) { + ChatOptions options = new ChatOptions(); + options.setStreaming(streaming); + ChatContext context = new ChatContext(); + context.setOptions(options); + return context; + } +} diff --git a/easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java b/easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java new file mode 100644 index 0000000..7895a92 --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/test/util/ImageUtilTest.java @@ -0,0 +1,36 @@ +package com.easyagents.core.test.util; + +import com.easyagents.core.message.UserMessage; +import com.easyagents.core.util.ImageUtil; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +/** + * 图片 Data URI 转换测试。 + */ +public class ImageUtilTest { + + /** + * 验证图片字节会生成符合标准的完整 Data URI。 + */ + @Test + public void shouldAddDataSchemeWhenEncodingImageBytes() { + String dataUri = ImageUtil.imageBytesToDataUri(new byte[]{1, 2, 3}, "image/png"); + + Assert.assertEquals("data:image/png;base64,AQID", dataUri); + } + + /** + * 验证用户消息添加图片字节时保留完整 Data URI。 + */ + @Test + public void shouldStoreCompleteDataUriInUserMessage() { + UserMessage message = new UserMessage(); + + message.addImageBytes(new byte[]{1, 2, 3}, "image/png"); + + Assert.assertEquals(List.of("data:image/png;base64,AQID"), message.getImageUrls()); + } +} diff --git a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java index 881ab7c..9ab2422 100644 --- a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java +++ b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java @@ -61,7 +61,7 @@ public class EasyAgentsLlm implements Llm { AiMessage aiMessage = response.getMessage(); if (aiMessage != null) { - return aiMessage.getContent(); + return aiMessage.getTextContent(); } throw new RuntimeException("EasyAgentsLlm can not get aiMessage!"); From 7e59f0e638c8dba31d1a6e55c4bf88cd1052c844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 23 Jul 2026 11:52:50 +0800 Subject: [PATCH 05/33] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E6=B5=81=E5=90=8C=E6=AD=A5=E6=89=A7=E8=A1=8C=E5=B9=B6?= =?UTF-8?q?=E5=8F=91=E7=9B=91=E5=90=AC=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用写时复制集合保证事件分发期间安全增删监听器 - 按工作流实例路由同步结果并补充并发回归测试 --- .../flow/core/chain/EventManager.java | 18 ++- .../core/chain/runtime/ChainExecutor.java | 84 +++++++++---- .../test/ChainExecutorConcurrencyTest.java | 112 ++++++++++++++++++ .../test/EventManagerConcurrencyTest.java | 91 ++++++++++++++ 4 files changed, 278 insertions(+), 27 deletions(-) create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/EventManagerConcurrencyTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EventManager.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EventManager.java index 50787cd..3955d38 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EventManager.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EventManager.java @@ -20,23 +20,31 @@ import com.easyagents.flow.core.chain.listener.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +/** + * 管理工作流执行过程中的事件、输出及错误监听器。 + * + *

监听器的注册与移除频率远低于事件分发频率,使用写时复制集合保证分发过程可以无锁遍历, + * 同时允许其他线程安全地注册或移除监听器。

+ */ public class EventManager { private static final Logger log = LoggerFactory.getLogger(EventManager.class); protected final Map, List> eventListeners = new ConcurrentHashMap<>(); - protected final List outputListeners = Collections.synchronizedList(new ArrayList<>()); - protected final List chainErrorListeners = Collections.synchronizedList(new ArrayList<>()); - protected final List nodeErrorListeners = Collections.synchronizedList(new ArrayList<>()); + protected final List outputListeners = new CopyOnWriteArrayList<>(); + protected final List chainErrorListeners = new CopyOnWriteArrayList<>(); + protected final List nodeErrorListeners = new CopyOnWriteArrayList<>(); /** * ---------- 通用事件监听器 ---------- */ public void addEventListener(Class eventClass, ChainEventListener listener) { - eventListeners.computeIfAbsent(eventClass, k -> Collections.synchronizedList(new ArrayList<>())).add(listener); + eventListeners.computeIfAbsent(eventClass, key -> new CopyOnWriteArrayList<>()).add(listener); } public void addEventListener(ChainEventListener listener) { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java index 3f6bdb2..a63bfa3 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java @@ -44,6 +44,9 @@ public class ChainExecutor { private final NodeStateRepository nodeStateRepository; private final TriggerScheduler triggerScheduler; private final EventManager eventManager = new EventManager(); + /** 等待同步执行结果的任务,按工作流实例 ID 进行常量时间路由。 */ + private final ConcurrentMap>> pendingExecutions = + new ConcurrentHashMap<>(); public ChainExecutor(ChainDefinitionRepository definitionRepository , ChainStateRepository chainStateRepository @@ -53,7 +56,7 @@ public class ChainExecutor { this.chainStateRepository = chainStateRepository; this.nodeStateRepository = nodeStateRepository; this.triggerScheduler = ChainRuntime.triggerScheduler(); - this.triggerScheduler.registerConsumer(this::accept); + registerRuntimeCallbacks(); } @@ -65,7 +68,7 @@ public class ChainExecutor { this.chainStateRepository = chainStateRepository; this.nodeStateRepository = nodeStateRepository; this.triggerScheduler = triggerScheduler; - this.triggerScheduler.registerConsumer(this::accept); + registerRuntimeCallbacks(); } @@ -79,26 +82,12 @@ public class ChainExecutor { String stateInstanceId = chain.getStateInstanceId(); CompletableFuture> future = new CompletableFuture<>(); - ChainEventListener listener = (event, c) -> { - if (event instanceof ChainStatusChangeEvent) { - if (((ChainStatusChangeEvent) event).getStatus().isTerminal() - && c.getStateInstanceId().equals(stateInstanceId)) { - ChainState state = chainStateRepository.load(stateInstanceId); - Map execResult = state.getExecuteResult(); - future.complete(execResult != null ? execResult : Collections.emptyMap()); - } - } - }; - - ChainErrorListener errorListener = (error, c) -> { - if (c.getStateInstanceId().equals(stateInstanceId)) { - future.completeExceptionally(error); - } - }; + CompletableFuture> existing = pendingExecutions.putIfAbsent(stateInstanceId, future); + if (existing != null) { + throw new IllegalStateException("Duplicate pending chain execution: " + stateInstanceId); + } try { - this.addEventListener(listener); - this.addErrorListener(errorListener); chain.start(variables); Map result = future.get(timeout, unit); clearDefaultStates(result); @@ -114,8 +103,59 @@ public class ChainExecutor { future.cancel(true); throw new RuntimeException("Execution failed", e.getCause()); } finally { - this.removeEventListener(listener); - this.removeErrorListener(errorListener); + pendingExecutions.remove(stateInstanceId, future); + } + } + + /** + * 注册工作流调度和同步结果路由回调。 + */ + private void registerRuntimeCallbacks() { + eventManager.addEventListener(ChainStatusChangeEvent.class, this::completePendingExecution); + eventManager.addChainErrorListener(this::failPendingExecution); + triggerScheduler.registerConsumer(this::accept); + } + + /** + * 在工作流进入终态时完成对应的同步等待任务。 + * + * @param event 工作流状态事件 + * @param chain 产生事件的工作流实例 + */ + private void completePendingExecution(Event event, Chain chain) { + if (!(event instanceof ChainStatusChangeEvent statusChangeEvent) + || !statusChangeEvent.getStatus().isTerminal()) { + return; + } + + String stateInstanceId = chain.getStateInstanceId(); + CompletableFuture> future = pendingExecutions.get(stateInstanceId); + if (future == null) { + return; + } + + try { + ChainState state = chainStateRepository.load(stateInstanceId); + if (state == null) { + throw new ChainException("Chain state not found: " + stateInstanceId); + } + Map execResult = state.getExecuteResult(); + future.complete(execResult != null ? execResult : Collections.emptyMap()); + } catch (Exception error) { + future.completeExceptionally(error); + } + } + + /** + * 将工作流执行异常传递给对应的同步等待任务。 + * + * @param error 工作流执行异常 + * @param chain 发生异常的工作流实例 + */ + private void failPendingExecution(Throwable error, Chain chain) { + CompletableFuture> future = pendingExecutions.get(chain.getStateInstanceId()); + if (future != null) { + future.completeExceptionally(error); } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java new file mode 100644 index 0000000..482736c --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java @@ -0,0 +1,112 @@ +/** + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + *

+ * Licensed under the GNU Lesser General Public License (LGPL) ,Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl-3.0.txt + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.Edge; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import com.easyagents.flow.core.node.EndNode; +import com.easyagents.flow.core.node.StartNode; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * {@link ChainExecutor} 并发同步执行测试。 + */ +public class ChainExecutorConcurrencyTest { + + /** + * 验证多个同步调用可以通过实例 ID 独立接收执行结果。 + * + * @throws Exception 并发任务执行失败、超时或中断时抛出 + */ + @Test + public void shouldRouteConcurrentSynchronousExecutionResults() throws Exception { + ScheduledExecutorService schedulerPool = Executors.newScheduledThreadPool(2); + ExecutorService workerPool = Executors.newFixedThreadPool(8); + ExecutorService callerPool = Executors.newFixedThreadPool(8); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + ChainDefinition definition = createDefinition(); + ChainExecutor chainExecutor = new ChainExecutor( + id -> definition, + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository(), + triggerScheduler); + + int executionCount = 16; + CountDownLatch startGate = new CountDownLatch(1); + List>> executions = new ArrayList<>(executionCount); + + try { + for (int index = 0; index < executionCount; index++) { + executions.add(callerPool.submit(() -> { + startGate.await(); + return chainExecutor.execute(definition.getId(), Collections.emptyMap()); + })); + } + startGate.countDown(); + + for (Future> execution : executions) { + Assert.assertNotNull(execution.get(10, TimeUnit.SECONDS)); + } + } finally { + startGate.countDown(); + callerPool.shutdownNow(); + triggerScheduler.shutdown(); + } + } + + /** + * 创建仅包含开始和结束节点的测试工作流。 + * + * @return 测试工作流定义 + */ + private ChainDefinition createDefinition() { + ChainDefinition definition = new ChainDefinition(); + definition.setId("concurrent-sync-test"); + + StartNode startNode = new StartNode(); + startNode.setId("start"); + EndNode endNode = new EndNode(); + endNode.setId("end"); + + Edge edge = new Edge(); + edge.setId("start-to-end"); + edge.setSource(startNode.getId()); + edge.setTarget(endNode.getId()); + + definition.addNode(startNode); + definition.addNode(endNode); + definition.addEdge(edge); + return definition; + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/EventManagerConcurrencyTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/EventManagerConcurrencyTest.java new file mode 100644 index 0000000..67279c2 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/EventManagerConcurrencyTest.java @@ -0,0 +1,91 @@ +/** + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + *

+ * Licensed under the GNU Lesser General Public License (LGPL) ,Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl-3.0.txt + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.EventManager; +import com.easyagents.flow.core.chain.event.ChainEndEvent; +import com.easyagents.flow.core.chain.listener.ChainEventListener; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * {@link EventManager} 并发行为测试。 + */ +public class EventManagerConcurrencyTest { + + /** + * 验证事件分发期间可以从其他线程移除监听器,当前分发使用稳定快照,后续分发不再调用已移除监听器。 + * + * @throws Exception 等待测试线程超时或中断时抛出 + */ + @Test + public void shouldAllowListenerRemovalDuringEventDispatch() throws Exception { + EventManager eventManager = new EventManager(); + CountDownLatch dispatchStarted = new CountDownLatch(1); + CountDownLatch continueDispatch = new CountDownLatch(1); + AtomicInteger retainedInvocationCount = new AtomicInteger(); + AtomicInteger removedInvocationCount = new AtomicInteger(); + AtomicReference dispatchFailure = new AtomicReference<>(); + + ChainEventListener blockingListener = (event, chain) -> { + dispatchStarted.countDown(); + try { + if (!continueDispatch.await(3, TimeUnit.SECONDS)) { + throw new AssertionError("事件分发等待超时"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("事件分发线程被中断", e); + } + }; + ChainEventListener retainedListener = (event, chain) -> retainedInvocationCount.incrementAndGet(); + ChainEventListener removedListener = (event, chain) -> removedInvocationCount.incrementAndGet(); + eventManager.addEventListener(blockingListener); + eventManager.addEventListener(retainedListener); + eventManager.addEventListener(removedListener); + + Thread dispatchThread = new Thread(() -> { + try { + eventManager.notifyEvent(new ChainEndEvent(null), null); + } catch (Throwable error) { + dispatchFailure.set(error); + } + }, "event-manager-dispatch-test"); + dispatchThread.start(); + + try { + Assert.assertTrue("事件分发应在测试超时前开始", dispatchStarted.await(3, TimeUnit.SECONDS)); + eventManager.removeEventListener(removedListener); + } finally { + continueDispatch.countDown(); + } + + dispatchThread.join(3000L); + Assert.assertFalse("事件分发线程应正常结束", dispatchThread.isAlive()); + Assert.assertNull("并发移除监听器不应中断事件分发", dispatchFailure.get()); + Assert.assertEquals(1, retainedInvocationCount.get()); + Assert.assertEquals("当前事件应按分发开始时的快照通知", 1, removedInvocationCount.get()); + + eventManager.notifyEvent(new ChainEndEvent(null), null); + Assert.assertEquals(2, retainedInvocationCount.get()); + Assert.assertEquals("后续事件不应通知已移除监听器", 1, removedInvocationCount.get()); + } +} From fbeece2d891fcf8da22861805ab2a88d9b9f23e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 23 Jul 2026 19:45:49 +0800 Subject: [PATCH 06/33] =?UTF-8?q?fix:=20=E5=AE=8C=E5=96=84=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=AE=A1=E6=89=B9=E8=B0=83=E7=94=A8=E7=BB=91=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 以 toolCallId、工具名称和入参绑定一次性执行授权 - 支持批次审批、重复调用去重及拒绝过期处理 - 补充多工具审批与授权消费回归测试 --- .../agent/runtime/AgentResumeRequest.java | 4 +- .../agentscope/AgentScopeReActRuntime.java | 103 ++-- .../interceptor/ToolHitlInterceptor.java | 276 ++++++++- .../hitl/AgentToolApprovalCoordinator.java | 526 +++++++++++++++++- .../hitl/AgentToolApprovalResolution.java | 93 ++++ .../AgentScopeStatefulRuntimeTest.java | 281 ++++++++++ .../AgentToolApprovalCoordinatorTest.java | 258 +++++++++ 7 files changed, 1454 insertions(+), 87 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalResolution.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentResumeRequest.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentResumeRequest.java index b0e7494..e042bfc 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentResumeRequest.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/AgentResumeRequest.java @@ -35,7 +35,9 @@ public class AgentResumeRequest { * *

该字段仅供服务端集成层使用。普通调用方不应设置该标记;设置后 runtime 会跳过 * 当前进程内 {@code AgentToolApprovalCoordinator} 的 token 存在性校验,用于服务重启或跨节点后 - * 从 AgentScope session 中继续 pending tool。

+ * 从 AgentScope session 中继续 pending tool。批准请求必须在 metadata 中提供 + * {@code toolCallId/toolName/toolInput},多个调用使用 {@code approvedToolCalls} 列表, + * 以便 runtime 将持久化审批结果绑定到实际工具调用。

*/ private boolean trusted; diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java index 7530e33..1dcbdef 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java @@ -9,7 +9,9 @@ import com.easyagents.agent.runtime.event.observer.AgentRuntimeErrorObserver; import com.easyagents.agent.runtime.event.observer.ReasoningLifecycleObserver; import com.easyagents.agent.runtime.event.observer.SkillExecutionObserver; import com.easyagents.agent.runtime.event.observer.ToolExecutionObserver; +import com.easyagents.agent.runtime.hitl.AgentPendingState; import com.easyagents.agent.runtime.hitl.AgentToolApprovalCoordinator; +import com.easyagents.agent.runtime.hitl.AgentToolApprovalResolution; import com.easyagents.agent.runtime.hitl.AgentToolApprovalRejectedException; import com.easyagents.agent.runtime.knowledge.AgentKnowledgeSpec; import com.easyagents.agent.runtime.knowledge.citation.AgentKnowledgeCitationMatcher; @@ -153,6 +155,9 @@ public class AgentScopeReActRuntime implements AgentRuntime { */ @Override public void close() { + if (approvalCoordinator != null) { + approvalCoordinator.cancelAll("Agent runtime has been closed."); + } closeMcpClients(); initialized.set(false); } @@ -188,17 +193,27 @@ public class AgentScopeReActRuntime implements AgentRuntime { return Flux.error(new AgentRuntimeException("Agent runtime is already streaming.")); } AgentRuntimeExecutionContext executionContext = createResumeExecutionContext(request); + AgentToolApprovalResolution resolution = null; try { - if (!request.isTrusted()) { - approvalCoordinator.consume(request); + if (request.isTrusted()) { + approvalCoordinator.authorizeTrustedExecution(request); + } else { + resolution = approvalCoordinator.resolve(request); } } catch (RuntimeException error) { running.set(false); throw error; } - // 审批拒绝 - if (!request.isApproved()) { - executionContext.setCancelReason(request.getRejectReason()); + if (resolution != null + && resolution.getStatus() == AgentToolApprovalResolution.Status.WAITING) { + return waitingForRemainingApprovals(executionContext, resolution); + } + if (!request.isApproved() + || resolution != null + && (resolution.getStatus() == AgentToolApprovalResolution.Status.REJECTED + || resolution.getStatus() == AgentToolApprovalResolution.Status.EXPIRED)) { + String cancelReason = resolution == null ? request.getRejectReason() : resolution.getReason(); + executionContext.setCancelReason(cancelReason); return Flux.defer(() -> { saveSession(); return Flux.just(started(executionContext), cancelled(executionContext)); @@ -240,8 +255,6 @@ public class AgentScopeReActRuntime implements AgentRuntime { AtomicReference finalMessage = new AtomicReference<>(); // HITL 暂停事件。被设置后,本轮以 SUSPENDED 挂起而不是 COMPLETED 结束。 AtomicReference suspendedEvent = new AtomicReference<>(); - // 本轮 HITL 待审批项来自旁路交互事件,最终会合并进 SUSPENDED 挂起事件。 - List> pendingApprovals = new CopyOnWriteArrayList<>(); // 知识库引注。 Map knowledgeReferences = new LinkedHashMap<>(); // 流式输出归一化,防止出现累计快照的重复输出。 @@ -250,8 +263,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { AtomicBoolean cancelled = new AtomicBoolean(false); // 旁线路监察事件流式输出。 Flux sideEventFlux = sideEvents.asFlux() - .doOnNext(event -> updateKnowledgeReferences(knowledgeReferences, event)) - .doOnNext(event -> updatePendingApprovals(pendingApprovals, event)); + .doOnNext(event -> updateKnowledgeReferences(knowledgeReferences, event)); // 主线路 agent 交互。resume 场景会传入空列表,让 AgentScope 从 pending tool 继续执行。 Flux mainEventFlux = agent.stream(inputSupplier.get(), streamOptions()) .timeout(executionContext.getAgentDefinition().getExecutionOptions().getTimeout()) @@ -271,9 +283,8 @@ public class AgentScopeReActRuntime implements AgentRuntime { .concatWith(Flux.defer(() -> { AgentRuntimeEvent suspended = suspendedEvent.get(); if (suspended != null) { - // 触发 hitl 审批事件,暂时挂起。 - suspended.getPayload().put("pendingApprovals", pendingApprovals); - return Flux.just(suspended); + // SUSPENDED 已在主线路中输出,结束阶段不再重复发送。 + return Flux.empty(); } return Flux.just(completed(executionContext, finalText.toString(), finalMessage.get(), knowledgeReferences)); @@ -542,9 +553,10 @@ public class AgentScopeReActRuntime implements AgentRuntime { if (sourceEvent.getMessage() != null) { event.setMessage(messageAdapter.toAgentMessage(sourceEvent.getMessage())); } - event.getPayload().put("reason", context.getMetadata().getOrDefault("hitlSuspendReason", "TOOL_APPROVAL_REQUIRED")); - Object pendingApprovals = context.getMetadata().get("hitlPendingApprovals"); - event.getPayload().put("pendingApprovals", pendingApprovals instanceof List list ? list : List.of()); + event.getPayload().put("reason", "TOOL_APPROVAL_REQUIRED"); + event.getPayload().put("pendingApprovals", approvalCoordinator.pendingStates(context.getSessionId()).stream() + .map(this::pendingApprovalPayload) + .toList()); event.getMetadata().put("source", "AGENTSCOPE_STREAM"); event.getMetadata().put("generateReason", sourceEvent.getMessage() == null ? GenerateReason.REASONING_STOP_REQUESTED.name() @@ -552,6 +564,46 @@ public class AgentScopeReActRuntime implements AgentRuntime { return event; } + /** + * 在同一审批批次仍有未决工具时返回挂起事件,并保持 AgentScope pending tools 不执行。 + * + * @param context 本轮恢复上下文 + * @param resolution 审批批次决议 + * @return 开始与挂起事件流 + */ + private Flux waitingForRemainingApprovals(AgentRuntimeExecutionContext context, + AgentToolApprovalResolution resolution) { + AgentRuntimeEvent suspended = base(context, AgentRuntimeEventType.SUSPENDED); + suspended.getPayload().put("reason", "TOOL_APPROVAL_REQUIRED"); + suspended.getPayload().put("pendingApprovals", resolution.getRemainingStates().stream() + .map(this::pendingApprovalPayload) + .toList()); + suspended.getMetadata().put("source", "APPROVAL_COORDINATOR"); + suspended.getMetadata().put("approvalStatus", AgentToolApprovalResolution.Status.WAITING.name()); + return Flux.just(started(context), suspended) + .doOnNext(event -> context.getConversationRecorder().record(context, event)) + .doFinally(signalType -> cleanupTurn()); + } + + /** + * 将待审批状态转换为前端可消费的稳定字段。 + * + * @param state 待审批状态 + * @return 待审批载荷 + */ + private Map pendingApprovalPayload(AgentPendingState state) { + Map payload = new LinkedHashMap<>(); + payload.put("resumeToken", state.getResumeToken().getValue()); + payload.put("toolCallId", state.getToolCallId()); + payload.put("toolName", state.getToolName()); + payload.put("toolInput", state.getToolInput()); + payload.put("input", state.getToolInput()); + payload.put("approvalPrompt", state.getApprovalPrompt()); + payload.put("approvalMetadata", state.getMetadata()); + payload.put("expiresAt", state.getExpiresAt() == null ? null : state.getExpiresAt().toString()); + return payload; + } + /** * 生成开始事件。 * @@ -742,6 +794,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { * 清理本轮状态。 */ private void cleanupTurn() { + approvalCoordinator.clearExecutionAuthorizations(); turnContextHolder.clear(); running.set(false); } @@ -822,26 +875,6 @@ public class AgentScopeReActRuntime implements AgentRuntime { } } - /** - * 从工具审批旁路事件中收集本轮待审批项。 - * - * @param pendingApprovals 待审批项集合 - * @param event 运行时事件 - */ - private void updatePendingApprovals(List> pendingApprovals, AgentRuntimeEvent event) { - if (event == null || event.getEventType() != AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) { - return; - } - Map approval = new LinkedHashMap<>(); - approval.put("resumeToken", event.getPayload().get("resumeToken")); - approval.put("toolCallId", event.getPayload().get("toolCallId")); - approval.put("toolName", event.getPayload().get("toolName")); - approval.put("toolInput", event.getPayload().get("toolInput")); - approval.put("expiresAt", event.getPayload().get("expiresAt")); - approval.put("approvalPrompt", event.getPayload().get("approvalPrompt")); - pendingApprovals.add(approval); - } - /** * 从知识库旁路事件中收集本轮候选引用。 * diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java index 4012f8e..b72bbc2 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java @@ -1,6 +1,7 @@ package com.easyagents.agent.runtime.event.interceptor; import com.easyagents.agent.runtime.AgentRuntimeExecutionContext; +import com.easyagents.agent.runtime.AgentRuntimeException; import com.easyagents.agent.runtime.event.AgentRuntimeEvent; import com.easyagents.agent.runtime.event.AgentRuntimeEventBridge; import com.easyagents.agent.runtime.event.AgentRuntimeEventType; @@ -11,6 +12,8 @@ import com.easyagents.agent.runtime.hitl.AgentToolApprovalRequest; import com.easyagents.agent.runtime.tool.AgentToolSpec; import io.agentscope.core.hook.HookEvent; import io.agentscope.core.hook.PostReasoningEvent; +import io.agentscope.core.hook.PreActingEvent; +import io.agentscope.core.message.ContentBlock; import io.agentscope.core.message.Msg; import io.agentscope.core.message.ToolUseBlock; import reactor.core.publisher.Mono; @@ -21,25 +24,28 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; /** * 工具 HITL 主线路干预器。 * - *

本 interceptor 专门处理“工具执行前人工审批”。监听 AgentScope 原生 - * {@link PostReasoningEvent}

+ *

本 interceptor 专门处理“工具执行前人工审批”。通过 AgentScope 原生 + * {@link PostReasoningEvent} 建立审批批次,并在 {@link PreActingEvent} 消费一次性执行授权。

* - *

这里包含两类动作: + *

这里包含三类动作: *

    *
  • 主线路干预:发现待审批工具后调用 {@link PostReasoningEvent#stopAgent()}, * 让 AgentScope 返回当前带 ToolUseBlock 的消息并暂停工具执行。
  • + *
  • 执行前校验:按工具调用身份消费一次性执行授权,阻止未批准或被篡改的调用。
  • *
  • 旁路交互事件:通过 {@link AgentRuntimeEventBridge} 发出 * {@link AgentRuntimeEventType#TOOL_APPROVAL_REQUIRED},通知调用方展示审批交互。
  • *
* - *

注意:本 interceptor 不执行工具、不写入 AgentScope memory/session,也不实现恢复。 - * 后续 resume 流程应基于 AgentScope pending tool 状态继续调用 agent stream/call。

+ *

注意:本 interceptor 不执行工具。后续 resume 流程应基于 AgentScope pending tool + * 状态继续调用 agent stream/call,实际工具执行仍由 AgentScope Toolkit 完成。

*/ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { @@ -76,6 +82,8 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { public Mono intercept(T event) { if (event instanceof PostReasoningEvent postReasoningEvent) { interceptPostReasoning(postReasoningEvent); + } else if (event instanceof PreActingEvent preActingEvent) { + interceptPreActing(preActingEvent); } return Mono.just(event); } @@ -93,47 +101,191 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { return 50; } + /** + * 归一化待审批调用、创建审批批次并暂停 AgentScope。 + * + * @param event 推理完成事件 + */ private void interceptPostReasoning(PostReasoningEvent event) { - Msg reasoningMessage = event.getReasoningMessage(); + Msg reasoningMessage = normalizeApprovalToolUses(event.getReasoningMessage()); if (reasoningMessage == null) { return; } + if (reasoningMessage != event.getReasoningMessage()) { + event.setReasoningMessage(reasoningMessage); + } List approvalRequiredTools = approvalRequiredTools(reasoningMessage); if (approvalRequiredTools.isEmpty()) { return; } - List> pendingApprovals = new ArrayList<>(); + String approvalBatchId = approvalBatchId(reasoningMessage); for (ToolUseBlock toolUse : approvalRequiredTools) { AgentToolSpec toolSpec = toolSpecs.get(toolUse.getName()); - AgentPendingState pendingState = registerPendingState(toolSpec, toolUse); + AgentPendingState pendingState = registerPendingState(toolSpec, toolUse, approvalBatchId); + if (pendingState.getEventId() != null && !pendingState.getEventId().isBlank()) { + continue; + } AgentRuntimeEvent approvalEvent = toolApprovalRequiredEvent(toolSpec, toolUse, pendingState); pendingState.setEventId(approvalEvent.getEventId()); - pendingApprovals.add(pendingApprovalPayload(pendingState, toolUse)); eventBridge.emit(approvalEvent); } - AgentRuntimeExecutionContext context = eventBridge.executionContext(); - if (context != null) { - context.getMetadata().put("hitlSuspended", true); - context.getMetadata().put("hitlSuspendReason", "TOOL_APPROVAL_REQUIRED"); - context.getMetadata().put("hitlPendingApprovals", pendingApprovals); - } event.stopAgent(); } + /** + * 在工具实际执行前消费与调用身份绑定的一次性授权。 + * + * @param event 工具执行前事件 + */ + private void interceptPreActing(PreActingEvent event) { + ToolUseBlock toolUse = event.getToolUse(); + AgentToolSpec toolSpec = toolUse == null ? null : toolSpecs.get(toolUse.getName()); + if (toolSpec == null || !toolSpec.isApprovalRequired()) { + return; + } + // 执行授权与 toolCallId、工具名称及入参同时绑定,并且只能消费一次。 + approvalCoordinator.consumeExecutionAuthorization( + toolUse.getId(), + toolUse.getName(), + toolUse.getInput()); + } + + /** + * 为缺少ID的审批调用补充稳定ID,并按 toolCallId 去除同轮重放调用。 + * + * @param reasoningMessage 原始推理消息 + * @return 归一化后的推理消息 + */ + private Msg normalizeApprovalToolUses(Msg reasoningMessage) { + if (reasoningMessage == null || reasoningMessage.getContent() == null + || reasoningMessage.getContent().isEmpty()) { + return reasoningMessage; + } + List normalizedContent = new ArrayList<>(reasoningMessage.getContent().size()); + Map seenApprovalCalls = new LinkedHashMap<>(); + boolean changed = false; + for (int contentIndex = 0; contentIndex < reasoningMessage.getContent().size(); contentIndex++) { + ContentBlock block = reasoningMessage.getContent().get(contentIndex); + if (!(block instanceof ToolUseBlock toolUse) || !isApprovalRequired(toolUse)) { + normalizedContent.add(block); + continue; + } + ToolUseBlock normalizedToolUse = toolUse; + if (toolUse.getId() == null || toolUse.getId().isBlank()) { + normalizedToolUse = copyWithId(toolUse, stableToolCallId(reasoningMessage, contentIndex)); + changed = true; + } + ToolCallSignature signature = new ToolCallSignature( + normalizedToolUse.getName(), normalizedToolUse.getInput()); + ToolCallSignature existing = seenApprovalCalls.putIfAbsent(normalizedToolUse.getId(), signature); + if (existing != null) { + if (!existing.equals(signature)) { + throw new AgentRuntimeException( + "Duplicate toolCallId is bound to a different tool call: " + normalizedToolUse.getId()); + } + // 相同 toolCallId 表示同一协议调用被重复返回,只保留第一次出现。 + changed = true; + continue; + } + normalizedContent.add(normalizedToolUse); + } + if (!changed) { + return reasoningMessage; + } + return Msg.builder() + .id(reasoningMessage.getId()) + .name(reasoningMessage.getName()) + .role(reasoningMessage.getRole()) + .content(normalizedContent) + .metadata(reasoningMessage.getMetadata()) + .timestamp(reasoningMessage.getTimestamp()) + .build(); + } + + /** + * 为缺少调用ID的工具生成跨同一推理消息重放稳定的调用ID。 + * + * @param reasoningMessage 推理消息 + * @param contentIndex 工具块在消息内容中的位置 + * @return 稳定工具调用ID + */ + private String stableToolCallId(Msg reasoningMessage, int contentIndex) { + String messageId = reasoningMessage.getId(); + if (messageId == null || messageId.isBlank()) { + return "hitl-" + UUID.randomUUID(); + } + return "hitl-" + messageId + "-" + contentIndex; + } + + /** + * 为同一推理消息生成稳定审批批次ID。 + * + * @param reasoningMessage 推理消息 + * @return 审批批次ID + */ + private String approvalBatchId(Msg reasoningMessage) { + String messageId = reasoningMessage == null ? null : reasoningMessage.getId(); + if (messageId == null || messageId.isBlank()) { + return "hitl-batch-" + UUID.randomUUID(); + } + return "hitl-batch-" + messageId; + } + + /** + * 判断工具调用是否要求人工审批。 + * + * @param toolUse 工具调用 + * @return 要求审批时为 true + */ + private boolean isApprovalRequired(ToolUseBlock toolUse) { + AgentToolSpec toolSpec = toolUse == null ? null : toolSpecs.get(toolUse.getName()); + return toolSpec != null && toolSpec.isApprovalRequired(); + } + + /** + * 复制工具调用并替换调用ID。 + * + * @param toolUse 原始工具调用 + * @param toolCallId 新工具调用ID + * @return 新工具调用 + */ + private ToolUseBlock copyWithId(ToolUseBlock toolUse, String toolCallId) { + return ToolUseBlock.builder() + .id(toolCallId) + .name(toolUse.getName()) + .input(toolUse.getInput()) + .content(toolUse.getContent()) + .metadata(toolUse.getMetadata()) + .build(); + } + + /** + * 获取推理消息中要求审批的工具调用。 + * + * @param reasoningMessage 推理消息 + * @return 待审批工具调用 + */ private List approvalRequiredTools(Msg reasoningMessage) { List toolUses = reasoningMessage.getContentBlocks(ToolUseBlock.class); if (toolUses == null || toolUses.isEmpty()) { return List.of(); } return toolUses.stream() - .filter(toolUse -> { - AgentToolSpec toolSpec = toolUse == null ? null : toolSpecs.get(toolUse.getName()); - return toolSpec != null && toolSpec.isApprovalRequired(); - }) + .filter(this::isApprovalRequired) .toList(); } - private AgentPendingState registerPendingState(AgentToolSpec toolSpec, ToolUseBlock toolUse) { + /** + * 注册审批批次中的待审批状态。 + * + * @param toolSpec 工具声明 + * @param toolUse 工具调用 + * @param approvalBatchId 审批批次ID + * @return 待审批状态 + */ + private AgentPendingState registerPendingState(AgentToolSpec toolSpec, + ToolUseBlock toolUse, + String approvalBatchId) { AgentRuntimeExecutionContext context = eventBridge.executionContext(); AgentToolApprovalRequest approvalRequest = toolSpec.getApprovalRequest(); Duration timeout = approvalRequest == null || approvalRequest.getTimeout() == null @@ -156,9 +308,18 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { approvalPrompt(approvalRequest), toolUse.getInput(), metadata, - Instant.now().plus(timeout)); + Instant.now().plus(timeout), + approvalBatchId); } + /** + * 构建工具审批请求事件。 + * + * @param toolSpec 工具声明 + * @param toolUse 工具调用 + * @param pendingState 待审批状态 + * @return 审批请求事件 + */ private AgentRuntimeEvent toolApprovalRequiredEvent(AgentToolSpec toolSpec, ToolUseBlock toolUse, AgentPendingState pendingState) { @@ -179,6 +340,13 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { return event; } + /** + * 构建待审批工具的稳定事件载荷。 + * + * @param pendingState 待审批状态 + * @param toolUse 工具调用 + * @return 待审批载荷 + */ private Map pendingApprovalPayload(AgentPendingState pendingState, ToolUseBlock toolUse) { Map payload = new LinkedHashMap<>(); payload.put("resumeToken", pendingState.getResumeToken().getValue()); @@ -191,6 +359,12 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { return payload; } + /** + * 将工具展示元数据补充到审批事件载荷。 + * + * @param payload 审批事件载荷 + * @param toolSpec 工具声明 + */ private void enrichToolPayload(Map payload, AgentToolSpec toolSpec) { if (toolSpec == null || toolSpec.getMetadata() == null || toolSpec.getMetadata().isEmpty()) { return; @@ -203,12 +377,25 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { putIfPresent(payload, metadata, "mcpTitle"); } + /** + * 在元数据包含指定字段时复制到事件载荷。 + * + * @param payload 事件载荷 + * @param metadata 工具元数据 + * @param key 字段名 + */ private void putIfPresent(Map payload, Map metadata, String key) { if (metadata.containsKey(key)) { payload.put(key, metadata.get(key)); } } + /** + * 获取审批提示文案。 + * + * @param approvalRequest 审批配置 + * @return 审批提示文案 + */ private String approvalPrompt(AgentToolApprovalRequest approvalRequest) { if (approvalRequest != null && approvalRequest.getApprovalPrompt() != null @@ -217,4 +404,51 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { } return "是否批准执行该工具?"; } + + /** + * 工具名称与输入组成的调用身份校验值。 + */ + private static final class ToolCallSignature { + private final String toolName; + private final Map toolInput; + + /** + * 创建工具调用身份校验值。 + * + * @param toolName 工具名称 + * @param toolInput 工具入参 + */ + private ToolCallSignature(String toolName, Map toolInput) { + this.toolName = toolName; + this.toolInput = toolInput == null ? Map.of() : new LinkedHashMap<>(toolInput); + } + + /** + * 比较工具调用语义是否一致。 + * + * @param object 待比较对象 + * @return 语义一致时为 true + */ + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof ToolCallSignature that)) { + return false; + } + return Objects.equals(toolName, that.toolName) + && Objects.equals(toolInput, that.toolInput); + } + + /** + * 计算工具调用语义哈希。 + * + * @return 哈希值 + */ + @Override + public int hashCode() { + return Objects.hash(toolName, toolInput); + } + } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java index 340b527..db1ca3e 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java @@ -5,18 +5,30 @@ import com.easyagents.agent.runtime.AgentRuntimeException; import reactor.core.publisher.Mono; import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; /** * 工具审批协调器。 */ public class AgentToolApprovalCoordinator { + /** 是否启用内存审批协调。 */ private final boolean enabled; - private final Map approvals = new ConcurrentHashMap<>(); + /** 恢复令牌到待审批项的索引。 */ + private final Map approvals = new LinkedHashMap<>(); + /** 审批批次ID到批次状态的索引。 */ + private final Map approvalBatches = new LinkedHashMap<>(); + /** 工具调用ID到恢复令牌的唯一索引。 */ + private final Map tokensByToolCallId = new LinkedHashMap<>(); + /** 工具调用ID到一次性执行授权的索引。 */ + private final Map executionAuthorizations = new LinkedHashMap<>(); /** * 创建已启用的协调器。 @@ -66,6 +78,54 @@ public class AgentToolApprovalCoordinator { Map toolInput, Map metadata, Instant expiresAt) { + return register(sessionId, agentId, toolCallId, toolName, approvalPrompt, toolInput, + metadata, expiresAt, null); + } + + /** + * 注册一个属于指定审批批次的待审批请求。 + * + *

同一批次内的全部工具调用均批准后,协调器才会签发执行授权。任意一项拒绝或 + * 过期都会关闭整个批次,避免未批准调用跟随已批准调用一起恢复执行。

+ * + * @param sessionId 会话ID + * @param agentId 智能体ID + * @param toolCallId 工具调用ID + * @param toolName 工具名称 + * @param approvalPrompt 审批文案 + * @param toolInput 工具入参 + * @param metadata 元数据 + * @param expiresAt 过期时间 + * @param approvalBatchId 审批批次ID;为空时创建单调用批次 + * @return 待审批状态 + */ + public synchronized AgentPendingState register(String sessionId, + String agentId, + String toolCallId, + String toolName, + String approvalPrompt, + Map toolInput, + Map metadata, + Instant expiresAt, + String approvalBatchId) { + if (enabled && (toolCallId == null || toolCallId.isBlank())) { + throw new AgentRuntimeException("Approval-required tool call must include toolCallId."); + } + if (enabled && toolCallId != null && !toolCallId.isBlank()) { + String existingToken = tokensByToolCallId.get(toolCallId); + PendingApproval existing = existingToken == null ? null : approvals.get(existingToken); + if (existing != null && !isExpired(existing.state)) { + if (!Objects.equals(existing.state.getToolName(), toolName) + || !Objects.equals(existing.state.getToolInput(), toolInput)) { + throw new AgentRuntimeException( + "Duplicate toolCallId is bound to a different tool call: " + toolCallId); + } + return existing.state; + } + if (existing != null) { + closeBatch(existing.batchId, "审批请求已过期。"); + } + } AgentPendingState state = new AgentPendingState(); state.setSessionId(sessionId); state.setAgentId(agentId); @@ -73,13 +133,26 @@ public class AgentToolApprovalCoordinator { state.setToolName(toolName); state.setApprovalPrompt(approvalPrompt); state.setToolInput(toolInput); - state.setMetadata(metadata); state.setExpiresAt(expiresAt); + String token = state.getResumeToken().getValue(); + String effectiveBatchId = approvalBatchId == null || approvalBatchId.isBlank() + ? token + : approvalBatchId; + Map effectiveMetadata = metadata == null + ? new LinkedHashMap<>() + : new LinkedHashMap<>(metadata); + effectiveMetadata.put("approvalBatchId", effectiveBatchId); + state.setMetadata(effectiveMetadata); if (enabled) { - String token = state.getResumeToken().getValue(); - PendingApproval pendingApproval = new PendingApproval(state, new CompletableFuture<>()); + PendingApproval pendingApproval = new PendingApproval( + state, effectiveBatchId, new CompletableFuture<>()); approvals.put(token, pendingApproval); - pendingApproval.future.whenComplete((response, error) -> approvals.remove(token)); + ApprovalBatch batch = approvalBatches.computeIfAbsent(effectiveBatchId, ApprovalBatch::new); + batch.tokens.add(token); + batch.members.put(token, pendingApproval); + if (toolCallId != null && !toolCallId.isBlank()) { + tokensByToolCallId.put(toolCallId, token); + } } return state; } @@ -90,7 +163,7 @@ public class AgentToolApprovalCoordinator { * @param resumeToken 恢复令牌 * @return 恢复请求 */ - public Mono await(AgentResumeToken resumeToken) { + public synchronized Mono await(AgentResumeToken resumeToken) { if (!enabled) { AgentResumeRequest request = new AgentResumeRequest(); request.setResumeToken(resumeToken); @@ -104,35 +177,211 @@ public class AgentToolApprovalCoordinator { if (pendingApproval == null) { return Mono.error(new AgentToolApprovalRejectedException("审批请求已失效。")); } + if (isExpired(pendingApproval.state)) { + closeBatch(pendingApproval.batchId, "审批请求已过期。"); + return Mono.error(new AgentToolApprovalRejectedException("审批请求已过期。")); + } return Mono.fromFuture(pendingApproval.future); } + /** + * 处理恢复请求并生成批次级审批决议。 + * + * @param request 恢复请求 + * @return 审批决议 + */ + public synchronized AgentToolApprovalResolution resolve(AgentResumeRequest request) { + validateResumeRequest(request); + if (!enabled) { + AgentPendingState state = new AgentPendingState(); + state.setResumeToken(request.getResumeToken()); + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.READY, state, List.of(), null); + } + String token = request.getResumeToken().getValue(); + PendingApproval pendingApproval = approvals.get(token); + if (pendingApproval == null || pendingApproval.decision != ApprovalDecision.PENDING) { + throw new AgentRuntimeException("Agent resume token is invalid, expired, or already consumed."); + } + if (isExpired(pendingApproval.state)) { + closeBatch(pendingApproval.batchId, "审批请求已过期。"); + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.EXPIRED, + pendingApproval.state, + List.of(), + "审批请求已过期。"); + } + + pendingApproval.decision = request.isApproved() + ? ApprovalDecision.APPROVED + : ApprovalDecision.REJECTED; + approvals.remove(token); + removeToolCallIndex(pendingApproval.state, token); + pendingApproval.future.complete(request); + + if (!request.isApproved()) { + String reason = request.getRejectReason() == null || request.getRejectReason().isBlank() + ? "工具执行已被拒绝。" + : request.getRejectReason(); + closeBatch(pendingApproval.batchId, reason); + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.REJECTED, + pendingApproval.state, + List.of(), + reason); + } + + ApprovalBatch batch = approvalBatches.get(pendingApproval.batchId); + if (batch == null) { + authorize(pendingApproval); + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.READY, + pendingApproval.state, + List.of(), + null); + } + List remainingStates = pendingStatesInBatch(batch); + AgentPendingState expiredState = remainingStates.stream() + .filter(this::isExpired) + .findFirst() + .orElse(null); + if (expiredState != null) { + closeBatch(batch.batchId, "审批请求已过期。"); + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.EXPIRED, + pendingApproval.state, + List.of(), + "审批请求已过期。"); + } + if (!remainingStates.isEmpty()) { + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.WAITING, + pendingApproval.state, + remainingStates, + null); + } + + for (String batchToken : batch.tokens) { + PendingApproval member = batch.members.get(batchToken); + if (member != null && member.decision == ApprovalDecision.APPROVED) { + authorize(member); + } + } + approvalBatches.remove(batch.batchId); + return new AgentToolApprovalResolution( + AgentToolApprovalResolution.Status.READY, + pendingApproval.state, + List.of(), + null); + } + /** * 消费恢复请求对应的待审批状态。 * - *

该方法用于有状态 runtime 的 HITL resume。第一版 pending state 仅保存在 - * 当前进程内存中,因此消费成功后会立即移除 token,避免重复恢复。

+ *

该兼容入口供工具适配器内部的 await/consume 流程使用。批次中仍有未决调用时 + * 会拒绝提前消费;单调用批准后会移除无需经过 PreActing 的执行凭证。

* * @param request 恢复请求 * @return 待审批状态 */ - public AgentPendingState consume(AgentResumeRequest request) { - if (request == null || request.getResumeToken() == null - || request.getResumeToken().getValue() == null - || request.getResumeToken().getValue().isBlank()) { - throw new AgentRuntimeException("Agent resume token is required."); + public synchronized AgentPendingState consume(AgentResumeRequest request) { + AgentToolApprovalResolution resolution = resolve(request); + if (resolution.getStatus() == AgentToolApprovalResolution.Status.WAITING) { + throw new AgentRuntimeException("Approval batch still has pending tool calls."); } + AgentPendingState state = resolution.getResolvedState(); + if (resolution.getStatus() == AgentToolApprovalResolution.Status.READY + && state != null + && state.getToolCallId() != null) { + // 兼容工具适配器内部 await/consume 流程,该流程会直接调用工具,不经过 PreActing 二次校验。 + executionAuthorizations.remove(state.getToolCallId()); + } + return state; + } + + /** + * 根据服务端持久化审批结果签发受信任的一次性执行授权。 + * + *

恢复元数据应提供单个 {@code toolCallId/toolName/toolInput},或通过 + * {@code approvedToolCalls} 提供上述字段组成的列表。该入口仅供已经完成持久化令牌 + * 校验和一次性消费的服务端集成层使用。

+ * + * @param request 受信任恢复请求 + */ + public synchronized void authorizeTrustedExecution(AgentResumeRequest request) { + validateResumeRequest(request); + if (!enabled || !request.isApproved()) { + return; + } + Map metadata = request.getMetadata() == null + ? Map.of() + : request.getMetadata(); + Object approvedToolCalls = metadata.get("approvedToolCalls"); + Map trustedAuthorizations = new LinkedHashMap<>(); + int authorizationCount = 0; + if (approvedToolCalls instanceof List calls) { + for (Object call : calls) { + if (call instanceof Map callMap) { + authorizeTrustedCall(callMap, trustedAuthorizations); + authorizationCount++; + } + } + } else if (metadata.containsKey("toolCallId")) { + authorizeTrustedCall(metadata, trustedAuthorizations); + authorizationCount++; + } + if (authorizationCount == 0) { + throw new AgentRuntimeException( + "Trusted resume metadata must include approved toolCallId, toolName, and toolInput."); + } + executionAuthorizations.putAll(trustedAuthorizations); + } + + /** + * 消费指定工具调用的一次性执行授权。 + * + * @param toolCallId 工具调用ID + * @param toolName 工具名称 + * @param toolInput 工具入参 + */ + public synchronized void consumeExecutionAuthorization(String toolCallId, + String toolName, + Map toolInput) { if (!enabled) { - AgentPendingState state = new AgentPendingState(); - state.setResumeToken(request.getResumeToken()); - return state; + return; } - PendingApproval pendingApproval = approvals.remove(request.getResumeToken().getValue()); - if (pendingApproval == null) { - throw new AgentRuntimeException("Agent resume token is invalid or expired."); + if (toolCallId == null || toolCallId.isBlank()) { + throw new AgentToolApprovalRejectedException("待执行工具缺少 toolCallId,无法校验审批结果。"); } - pendingApproval.future.complete(request); - return pendingApproval.state; + ExecutionAuthorization authorization = executionAuthorizations.remove(toolCallId); + if (authorization == null) { + throw new AgentToolApprovalRejectedException("工具调用未获得批准或批准已被消费。"); + } + if (!Objects.equals(authorization.toolName, toolName) + || !Objects.equals(authorization.toolInput, toolInput)) { + throw new AgentToolApprovalRejectedException("工具调用与已批准内容不一致。"); + } + } + + /** + * 清理尚未消费的一次性执行授权。 + */ + public synchronized void clearExecutionAuthorizations() { + executionAuthorizations.clear(); + } + + /** + * 获取指定会话当前仍待处理的审批状态。 + * + * @param sessionId 会话ID + * @return 待审批状态快照 + */ + public synchronized List pendingStates(String sessionId) { + return approvals.values().stream() + .filter(pending -> pending.decision == ApprovalDecision.PENDING) + .map(pending -> pending.state) + .filter(state -> sessionId == null || Objects.equals(sessionId, state.getSessionId())) + .toList(); } /** @@ -140,18 +389,16 @@ public class AgentToolApprovalCoordinator { * * @param reason 取消原因 */ - public void cancelAll(String reason) { + public synchronized void cancelAll(String reason) { if (!enabled) { return; } - for (PendingApproval pendingApproval : approvals.values()) { - AgentResumeRequest request = new AgentResumeRequest(); - request.setResumeToken(pendingApproval.state.getResumeToken()); - request.setApproved(false); - request.setRejectReason(reason); - pendingApproval.future.complete(request); + for (String batchId : new ArrayList<>(approvalBatches.keySet())) { + closeBatch(batchId, reason); } approvals.clear(); + tokensByToolCallId.clear(); + executionAuthorizations.clear(); } /** @@ -163,13 +410,232 @@ public class AgentToolApprovalCoordinator { return enabled; } + /** + * 校验恢复请求中的令牌字段。 + * + * @param request 恢复请求 + */ + private void validateResumeRequest(AgentResumeRequest request) { + if (request == null || request.getResumeToken() == null + || request.getResumeToken().getValue() == null + || request.getResumeToken().getValue().isBlank()) { + throw new AgentRuntimeException("Agent resume token is required."); + } + } + + /** + * 判断待审批状态是否已经过期。 + * + * @param state 待审批状态 + * @return 已过期时为 true + */ + private boolean isExpired(AgentPendingState state) { + return state != null + && state.getExpiresAt() != null + && !state.getExpiresAt().isAfter(Instant.now()); + } + + /** + * 获取批次内尚未决策的审批状态。 + * + * @param batch 审批批次 + * @return 未决审批状态 + */ + private List pendingStatesInBatch(ApprovalBatch batch) { + List states = new ArrayList<>(); + for (String token : batch.tokens) { + PendingApproval member = batch.members.get(token); + if (member != null && member.decision == ApprovalDecision.PENDING) { + states.add(member.state); + } + } + return states; + } + + /** + * 为已批准状态签发一次性执行授权。 + * + * @param pendingApproval 已批准状态 + */ + private void authorize(PendingApproval pendingApproval) { + AgentPendingState state = pendingApproval.state; + if (state.getToolCallId() == null || state.getToolCallId().isBlank()) { + throw new AgentRuntimeException("Approved tool call is missing toolCallId."); + } + executionAuthorizations.put(state.getToolCallId(), new ExecutionAuthorization( + state.getToolName(), + state.getToolInput())); + } + + /** + * 为服务端持久化审批结果签发一次性执行授权。 + * + * @param callMap 已批准调用元数据 + * @param trustedAuthorizations 本次恢复待签发的临时授权集合 + */ + private void authorizeTrustedCall(Map callMap, + Map trustedAuthorizations) { + String toolCallId = stringValue(callMap.get("toolCallId")); + String toolName = stringValue(callMap.get("toolName")); + if (toolCallId == null || toolName == null) { + throw new AgentRuntimeException( + "Trusted resume metadata must include non-empty toolCallId and toolName."); + } + Map toolInput = stringKeyMap(callMap.get("toolInput")); + ExecutionAuthorization authorization = new ExecutionAuthorization(toolName, toolInput); + ExecutionAuthorization previous = trustedAuthorizations.put(toolCallId, authorization); + if (previous != null + && (!Objects.equals(previous.toolName, toolName) + || !Objects.equals(previous.toolInput, toolInput))) { + throw new AgentRuntimeException( + "Trusted resume contains conflicting tool calls for toolCallId: " + toolCallId); + } + } + + /** + * 将值转换为非空字符串。 + * + * @param value 原始值 + * @return 非空字符串;无法转换时返回 null + */ + private String stringValue(Object value) { + if (value == null || String.valueOf(value).isBlank()) { + return null; + } + return String.valueOf(value); + } + + /** + * 将任意键 Map 转换为字符串键 Map。 + * + * @param value 原始值 + * @return 字符串键 Map + */ + private Map stringKeyMap(Object value) { + if (value == null) { + return Map.of(); + } + if (!(value instanceof Map source)) { + throw new AgentRuntimeException("Trusted resume toolInput must be a map."); + } + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : source.entrySet()) { + if (entry.getKey() != null) { + result.put(String.valueOf(entry.getKey()), entry.getValue()); + } + } + return result; + } + + /** + * 移除工具调用ID到审批令牌的索引。 + * + * @param state 待审批状态 + * @param token 审批令牌 + */ + private void removeToolCallIndex(AgentPendingState state, String token) { + if (state.getToolCallId() != null && !state.getToolCallId().isBlank()) { + tokensByToolCallId.remove(state.getToolCallId(), token); + } + } + + /** + * 关闭审批批次并拒绝尚未决策的审批项。 + * + * @param batchId 审批批次ID + * @param reason 关闭原因 + */ + private void closeBatch(String batchId, String reason) { + ApprovalBatch batch = approvalBatches.remove(batchId); + if (batch == null) { + return; + } + for (String token : batch.tokens) { + PendingApproval member = batch.members.get(token); + if (member == null) { + continue; + } + approvals.remove(token); + removeToolCallIndex(member.state, token); + if (member.decision == ApprovalDecision.PENDING) { + member.decision = ApprovalDecision.REJECTED; + AgentResumeRequest rejection = new AgentResumeRequest(); + rejection.setResumeToken(member.state.getResumeToken()); + rejection.setApproved(false); + rejection.setRejectReason(reason); + member.future.complete(rejection); + } + } + } + + /** + * 单个审批项的内部决策状态。 + */ + private enum ApprovalDecision { + PENDING, + APPROVED, + REJECTED + } + + /** + * 待审批项及其异步等待句柄。 + */ private static class PendingApproval { private final AgentPendingState state; + private final String batchId; private final CompletableFuture future; + private ApprovalDecision decision = ApprovalDecision.PENDING; - private PendingApproval(AgentPendingState state, CompletableFuture future) { + /** + * 创建待审批项。 + * + * @param state 待审批状态 + * @param batchId 审批批次ID + * @param future 审批响应等待句柄 + */ + private PendingApproval(AgentPendingState state, + String batchId, + CompletableFuture future) { this.state = Objects.requireNonNull(state, "state"); + this.batchId = Objects.requireNonNull(batchId, "batchId"); this.future = Objects.requireNonNull(future, "future"); } } + + /** + * 同一轮推理产生的审批批次。 + */ + private static class ApprovalBatch { + private final String batchId; + private final Set tokens = new LinkedHashSet<>(); + private final Map members = new LinkedHashMap<>(); + + /** + * 创建审批批次。 + * + * @param batchId 审批批次ID + */ + private ApprovalBatch(String batchId) { + this.batchId = Objects.requireNonNull(batchId, "batchId"); + } + } + + /** + * 已批准工具调用的一次性执行凭证。 + */ + private static class ExecutionAuthorization { + private final String toolName; + private final Map toolInput; + + /** + * 创建一次性执行授权。 + * + * @param toolName 工具名称 + * @param toolInput 工具入参 + */ + private ExecutionAuthorization(String toolName, Map toolInput) { + this.toolName = toolName; + this.toolInput = toolInput == null ? Map.of() : new LinkedHashMap<>(toolInput); + } + } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalResolution.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalResolution.java new file mode 100644 index 0000000..c67e72c --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalResolution.java @@ -0,0 +1,93 @@ +package com.easyagents.agent.runtime.hitl; + +import java.util.List; + +/** + * 工具审批决议。 + */ +public final class AgentToolApprovalResolution { + + /** + * 审批决议状态。 + */ + public enum Status { + /** + * 当前审批批次仍有待处理调用。 + */ + WAITING, + + /** + * 当前审批批次已全部批准,可以恢复执行。 + */ + READY, + + /** + * 当前审批批次已被拒绝。 + */ + REJECTED, + + /** + * 当前审批批次已过期。 + */ + EXPIRED + } + + private final Status status; + private final AgentPendingState resolvedState; + private final List remainingStates; + private final String reason; + + /** + * 创建工具审批决议。 + * + * @param status 决议状态 + * @param resolvedState 本次处理的待审批状态 + * @param remainingStates 同一批次剩余的待审批状态 + * @param reason 拒绝或过期原因 + */ + public AgentToolApprovalResolution(Status status, + AgentPendingState resolvedState, + List remainingStates, + String reason) { + this.status = status; + this.resolvedState = resolvedState; + this.remainingStates = remainingStates == null ? List.of() : List.copyOf(remainingStates); + this.reason = reason; + } + + /** + * 获取决议状态。 + * + * @return 决议状态 + */ + public Status getStatus() { + return status; + } + + /** + * 获取本次处理的待审批状态。 + * + * @return 待审批状态 + */ + public AgentPendingState getResolvedState() { + return resolvedState; + } + + /** + * 获取同一批次剩余的待审批状态。 + * + * @return 剩余待审批状态 + */ + public List getRemainingStates() { + return remainingStates; + } + + /** + * 获取拒绝或过期原因。 + * + * @return 原因 + */ + public String getReason() { + return reason; + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java index 50028ae..c4543e4 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java @@ -56,6 +56,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; /** @@ -843,6 +844,9 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertFalse(((List) suspended.getPayload().get("pendingApprovals")).isEmpty()); Assert.assertFalse(events.stream().anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); Assert.assertFalse(events.stream().anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_RESULT)); + Assert.assertEquals(1, events.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED) + .count()); } @Test(expected = AgentRuntimeException.class) @@ -927,6 +931,267 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertTrue(sessionStore.exists("session-1")); } + /** + * 验证同一轮推理包含多个审批工具时,全部批准前不会执行任何工具。 + */ + @Test + public void shouldWaitForAllToolApprovalsBeforeExecutingBatch() { + AgentInitRequest request = initRequest(); + AgentToolSpec searchSpec = new AgentToolSpec(); + searchSpec.setName("search"); + searchSpec.setDescription("search"); + searchSpec.setApprovalRequired(true); + AgentToolSpec auditSpec = new AgentToolSpec(); + auditSpec.setName("audit"); + auditSpec.setDescription("audit"); + auditSpec.setApprovalRequired(true); + request.getAgentDefinition().setToolSpecs(List.of(searchSpec, auditSpec)); + AtomicInteger invocationCount = new AtomicInteger(); + request.setToolInvokers(Map.of( + "search", (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("search result"); + }, + "audit", (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("audit result"); + })); + AgentScopeReActRuntime runtime = runtimeWithModel(List.of( + ChatResponse.builder() + .id("tool-call-message") + .content(List.of( + ToolUseBlock.builder() + .id("call-search") + .name("search") + .input(Map.of("q", "easyflow")) + .build(), + ToolUseBlock.builder() + .id("call-audit") + .name("audit") + .input(Map.of("scope", "current")) + .build())) + .finishReason("tool_calls") + .build(), + ChatResponse.builder() + .id("final-message") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build())); + runtime.init(request); + + List initialEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "use tools")) + .collectList() + .block(); + List approvals = initialEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .toList(); + + Assert.assertEquals(2, approvals.size()); + Assert.assertEquals(1, initialEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED) + .count()); + List firstResumeEvents = runtime.resume(resumeFromApproval(approvals.get(0), true)) + .collectList() + .block(); + + Assert.assertEquals(0, invocationCount.get()); + AgentRuntimeEvent waitingEvent = firstResumeEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED) + .findFirst() + .orElseThrow(); + Assert.assertEquals(1, ((List) waitingEvent.getPayload().get("pendingApprovals")).size()); + Assert.assertFalse(firstResumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_CALL)); + + List secondResumeEvents = runtime.resume(resumeFromApproval(approvals.get(1), true)) + .collectList() + .block(); + + Assert.assertEquals(2, invocationCount.get()); + Assert.assertEquals(2, secondResumeEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_RESULT) + .count()); + Assert.assertTrue(secondResumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); + } + + /** + * 验证不同 toolCallId 即使名称和入参相同也会分别审批和执行。 + */ + @Test + public void shouldPreserveDistinctApprovedToolCallsWithIdenticalInput() { + AgentInitRequest request = initRequest(); + AgentToolSpec toolSpec = new AgentToolSpec(); + toolSpec.setName("search"); + toolSpec.setDescription("search"); + toolSpec.setApprovalRequired(true); + request.getAgentDefinition().setToolSpecs(List.of(toolSpec)); + AtomicInteger invocationCount = new AtomicInteger(); + request.setToolInvokers(Map.of("search", (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("tool result"); + })); + Map input = Map.of("q", "easyflow"); + AgentScopeReActRuntime runtime = runtimeWithModel(List.of( + ChatResponse.builder() + .id("tool-call-message") + .content(List.of( + ToolUseBlock.builder() + .id("call-search") + .name("search") + .input(input) + .build(), + ToolUseBlock.builder() + .id("call-search-duplicate") + .name("search") + .input(input) + .build())) + .finishReason("tool_calls") + .build(), + ChatResponse.builder() + .id("final-message") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build())); + runtime.init(request); + + List suspendedEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "use tool")) + .collectList() + .block(); + List approvals = suspendedEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .toList(); + + Assert.assertEquals(2, approvals.size()); + List firstResumeEvents = runtime.resume(resumeFromApproval(approvals.get(0), true)) + .collectList() + .block(); + + Assert.assertEquals(0, invocationCount.get()); + Assert.assertTrue(firstResumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED)); + List secondResumeEvents = runtime.resume(resumeFromApproval(approvals.get(1), true)) + .collectList() + .block(); + + Assert.assertEquals(2, invocationCount.get()); + Assert.assertEquals(2, secondResumeEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_RESULT) + .count()); + Assert.assertTrue(secondResumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); + } + + /** + * 验证相同 toolCallId 的同轮重放只审批并执行一次。 + */ + @Test + public void shouldDeduplicateRepeatedToolCallIdWithinReasoning() { + AgentInitRequest request = initRequest(); + AgentToolSpec toolSpec = new AgentToolSpec(); + toolSpec.setName("search"); + toolSpec.setDescription("search"); + toolSpec.setApprovalRequired(true); + request.getAgentDefinition().setToolSpecs(List.of(toolSpec)); + AtomicInteger invocationCount = new AtomicInteger(); + request.setToolInvokers(Map.of("search", (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("tool result"); + })); + Map input = Map.of("q", "easyflow"); + AgentScopeReActRuntime runtime = runtimeWithModel(List.of( + ChatResponse.builder() + .id("tool-call-message") + .content(List.of( + ToolUseBlock.builder() + .id("call-search") + .name("search") + .input(input) + .build(), + ToolUseBlock.builder() + .id("call-search") + .name("search") + .input(input) + .build())) + .finishReason("tool_calls") + .build(), + ChatResponse.builder() + .id("final-message") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build())); + runtime.init(request); + + List suspendedEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "use tool")) + .collectList() + .block(); + List approvals = suspendedEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .toList(); + + Assert.assertEquals(1, approvals.size()); + List resumeEvents = runtime.resume(resumeFromApproval(approvals.get(0), true)) + .collectList() + .block(); + + Assert.assertEquals(1, invocationCount.get()); + Assert.assertEquals(1, resumeEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_RESULT) + .count()); + Assert.assertTrue(resumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); + } + + /** + * 验证模型未提供 toolCallId 时审批链路仍会获得完整的调用身份。 + */ + @Test + public void shouldProvideApprovalIdentityWhenModelOmitsToolCallId() { + AgentInitRequest request = initRequest(); + AgentToolSpec toolSpec = new AgentToolSpec(); + toolSpec.setName("search"); + toolSpec.setDescription("search"); + toolSpec.setApprovalRequired(true); + request.getAgentDefinition().setToolSpecs(List.of(toolSpec)); + request.setToolInvokers(Map.of("search", (arguments, context) -> + AgentToolResult.success("tool result"))); + AgentScopeReActRuntime runtime = runtimeWithModel(List.of( + ChatResponse.builder() + .id("tool-call-message") + .content(List.of(ToolUseBlock.builder() + .name("search") + .input(Map.of("q", "easyflow")) + .build())) + .finishReason("tool_calls") + .build(), + ChatResponse.builder() + .id("final-message") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build())); + runtime.init(request); + + List events = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "use tool")) + .collectList() + .block(); + AgentRuntimeEvent approval = events.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .findFirst() + .orElseThrow(); + + String toolCallId = String.valueOf(approval.getPayload().get("toolCallId")); + Assert.assertFalse(toolCallId.isBlank()); + @SuppressWarnings("unchecked") + Map metadata = + (Map) approval.getPayload().get("approvalMetadata"); + String approvalBatchId = String.valueOf(metadata.get("approvalBatchId")); + Assert.assertFalse(approvalBatchId.isBlank()); + } + @Test public void shouldCancelRejectedToolResumeWithoutExecutingTool() { InMemoryAgentSessionStore sessionStore = new InMemoryAgentSessionStore(); @@ -1105,6 +1370,22 @@ public class AgentScopeStatefulRuntimeTest { new AgentScopeMessageAdapter()); } + /** + * 根据审批事件创建恢复请求。 + * + * @param approvalEvent 工具审批事件 + * @param approved 是否批准 + * @return 恢复请求 + */ + private AgentResumeRequest resumeFromApproval(AgentRuntimeEvent approvalEvent, boolean approved) { + AgentResumeRequest request = new AgentResumeRequest(); + AgentResumeToken token = new AgentResumeToken(); + token.setValue(String.valueOf(approvalEvent.getPayload().get("resumeToken"))); + request.setResumeToken(token); + request.setApproved(approved); + return request; + } + private static class ScriptedModel implements Model { private final String modelName; diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java new file mode 100644 index 0000000..c3aad9d --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java @@ -0,0 +1,258 @@ +package com.easyagents.agent.runtime.hitl; + +import com.easyagents.agent.runtime.AgentResumeRequest; +import com.easyagents.agent.runtime.AgentRuntimeException; +import org.junit.Assert; +import org.junit.Test; + +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * 测试工具审批协调器。 + */ +public class AgentToolApprovalCoordinatorTest { + + /** + * 验证同批次全部调用批准后才签发逐调用执行授权。 + */ + @Test + public void shouldAuthorizeBatchOnlyAfterAllCallsApproved() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentPendingState first = register(coordinator, "call-1", "search", Map.of("q", "first"), "batch-1"); + AgentPendingState second = register(coordinator, "call-2", "search", Map.of("q", "second"), "batch-1"); + + AgentToolApprovalResolution firstResolution = coordinator.resolve(resume(first, true)); + + Assert.assertEquals(AgentToolApprovalResolution.Status.WAITING, firstResolution.getStatus()); + Assert.assertEquals(1, firstResolution.getRemainingStates().size()); + assertAuthorizationRejected(coordinator, "call-1", "search", Map.of("q", "first")); + + AgentToolApprovalResolution secondResolution = coordinator.resolve(resume(second, true)); + + Assert.assertEquals(AgentToolApprovalResolution.Status.READY, secondResolution.getStatus()); + coordinator.consumeExecutionAuthorization("call-1", "search", Map.of("q", "first")); + coordinator.consumeExecutionAuthorization("call-2", "search", Map.of("q", "second")); + assertAuthorizationRejected(coordinator, "call-1", "search", Map.of("q", "first")); + } + + /** + * 验证拒绝一个调用会关闭整个审批批次。 + */ + @Test + public void shouldRejectWholeBatchWhenAnyCallRejected() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentPendingState first = register(coordinator, "call-1", "search", Map.of("q", "first"), "batch-1"); + AgentPendingState second = register(coordinator, "call-2", "search", Map.of("q", "second"), "batch-1"); + coordinator.resolve(resume(first, true)); + + AgentResumeRequest rejection = resume(second, false); + rejection.setRejectReason("not allowed"); + AgentToolApprovalResolution resolution = coordinator.resolve(rejection); + + Assert.assertEquals(AgentToolApprovalResolution.Status.REJECTED, resolution.getStatus()); + Assert.assertEquals("not allowed", resolution.getReason()); + assertAuthorizationRejected(coordinator, "call-1", "search", Map.of("q", "first")); + try { + coordinator.resolve(resume(first, true)); + Assert.fail("已消费的审批令牌不能重复使用"); + } catch (AgentRuntimeException expected) { + Assert.assertTrue(expected.getMessage().contains("invalid")); + } + } + + /** + * 验证过期令牌不能签发工具执行授权。 + */ + @Test + public void shouldExpireApprovalBeforeResolution() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentPendingState expired = coordinator.register( + "session-1", + "agent-1", + "call-expired", + "search", + "approve", + Map.of("q", "expired"), + Map.of(), + Instant.now().minusSeconds(1), + "batch-expired"); + + AgentToolApprovalResolution resolution = coordinator.resolve(resume(expired, true)); + + Assert.assertEquals(AgentToolApprovalResolution.Status.EXPIRED, resolution.getStatus()); + assertAuthorizationRejected(coordinator, "call-expired", "search", Map.of("q", "expired")); + } + + /** + * 验证并发重复点击同一令牌时最多一个请求可以成功消费。 + * + * @throws Exception 并发任务执行失败时抛出 + */ + @Test + public void shouldConsumeConcurrentDuplicateApprovalOnlyOnce() throws Exception { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentPendingState pending = register( + coordinator, "call-1", "search", Map.of("q", "easyflow"), "batch-1"); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future first = executor.submit(() -> resolveAfter(start, coordinator, pending)); + Future second = executor.submit(() -> resolveAfter(start, coordinator, pending)); + start.countDown(); + + int successCount = (first.get() ? 1 : 0) + (second.get() ? 1 : 0); + + Assert.assertEquals(1, successCount); + coordinator.consumeExecutionAuthorization( + "call-1", "search", Map.of("q", "easyflow")); + assertAuthorizationRejected( + coordinator, "call-1", "search", Map.of("q", "easyflow")); + } finally { + executor.shutdownNow(); + } + } + + /** + * 验证工具名称或入参变化时批准凭证立即失效。 + */ + @Test + public void shouldRejectExecutionWhenApprovedCallIsModified() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentPendingState pending = register( + coordinator, "call-1", "search", Map.of("q", "easyflow"), "batch-1"); + coordinator.resolve(resume(pending, true)); + + assertAuthorizationRejected( + coordinator, "call-1", "search", Map.of("q", "modified")); + assertAuthorizationRejected( + coordinator, "call-1", "search", Map.of("q", "easyflow")); + } + + /** + * 验证跨节点受信任恢复仍需绑定明确的工具调用信息。 + */ + @Test + public void shouldAuthorizeTrustedExecutionByToolCallIdentity() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentResumeRequest request = new AgentResumeRequest(); + AgentResumeToken token = new AgentResumeToken(); + token.setValue("persisted-token"); + request.setResumeToken(token); + request.setApproved(true); + request.setTrusted(true); + request.setMetadata(Map.of( + "toolCallId", "call-1", + "toolName", "search", + "toolInput", Map.of("q", "easyflow"))); + + coordinator.authorizeTrustedExecution(request); + + coordinator.consumeExecutionAuthorization( + "call-1", "search", Map.of("q", "easyflow")); + assertAuthorizationRejected( + coordinator, "call-1", "search", Map.of("q", "easyflow")); + } + + /** + * 验证同一 toolCallId 不能被重新绑定到不同工具内容。 + */ + @Test + public void shouldRejectDuplicateToolCallIdWithDifferentInput() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + register(coordinator, "call-1", "search", Map.of("q", "easyflow"), "batch-1"); + + try { + register(coordinator, "call-1", "search", Map.of("q", "modified"), "batch-2"); + Assert.fail("重复 toolCallId 不能绑定不同入参"); + } catch (AgentRuntimeException expected) { + Assert.assertTrue(expected.getMessage().contains("Duplicate toolCallId")); + } + } + + /** + * 注册测试用审批状态。 + * + * @param coordinator 审批协调器 + * @param toolCallId 工具调用ID + * @param toolName 工具名称 + * @param toolInput 工具入参 + * @param batchId 审批批次ID + * @return 待审批状态 + */ + private AgentPendingState register(AgentToolApprovalCoordinator coordinator, + String toolCallId, + String toolName, + Map toolInput, + String batchId) { + return coordinator.register( + "session-1", + "agent-1", + toolCallId, + toolName, + "approve", + toolInput, + Map.of(), + Instant.now().plusSeconds(60), + batchId); + } + + /** + * 创建测试用恢复请求。 + * + * @param state 待审批状态 + * @param approved 是否批准 + * @return 恢复请求 + */ + private AgentResumeRequest resume(AgentPendingState state, boolean approved) { + AgentResumeRequest request = new AgentResumeRequest(); + request.setResumeToken(state.getResumeToken()); + request.setApproved(approved); + return request; + } + + /** + * 等待并发起跑信号后消费审批令牌。 + * + * @param start 起跑信号 + * @param coordinator 审批协调器 + * @param pending 待审批状态 + * @return 成功消费时为 true + * @throws InterruptedException 等待被中断时抛出 + */ + private boolean resolveAfter(CountDownLatch start, + AgentToolApprovalCoordinator coordinator, + AgentPendingState pending) throws InterruptedException { + start.await(); + try { + coordinator.resolve(resume(pending, true)); + return true; + } catch (AgentRuntimeException expected) { + return false; + } + } + + /** + * 断言工具执行授权不可用。 + * + * @param coordinator 审批协调器 + * @param toolCallId 工具调用ID + * @param toolName 工具名称 + * @param toolInput 工具入参 + */ + private void assertAuthorizationRejected(AgentToolApprovalCoordinator coordinator, + String toolCallId, + String toolName, + Map toolInput) { + try { + coordinator.consumeExecutionAuthorization(toolCallId, toolName, toolInput); + Assert.fail("未授权或已消费的工具调用必须被拒绝"); + } catch (AgentToolApprovalRejectedException expected) { + Assert.assertNotNull(expected.getMessage()); + } + } +} From 5b6b2db5d89606cade30b889bf35fb5ff69928c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 23 Jul 2026 19:46:07 +0800 Subject: [PATCH 07/33] =?UTF-8?q?fix:=20=E4=BF=9D=E7=95=99=E6=B5=81?= =?UTF-8?q?=E5=BC=8F=E8=BE=93=E5=87=BA=E8=BF=9E=E7=BB=AD=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E5=AD=97=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 按 AgentScope 增量协议原样保留普通文本分片 - 仅在终态完整快照中去除已发送前缀 - 补充连续端口字符回归测试 --- .../agentscope/AgentScopeReActRuntime.java | 20 ++-- .../AgentScopeStatefulRuntimeTest.java | 109 ++++++++++++++++++ 2 files changed, 121 insertions(+), 8 deletions(-) diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java index 1dcbdef..b59265b 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java @@ -257,7 +257,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { AtomicReference suspendedEvent = new AtomicReference<>(); // 知识库引注。 Map knowledgeReferences = new LinkedHashMap<>(); - // 流式输出归一化,防止出现累计快照的重复输出。 + // 按 AgentScope 增量协议累计正文,并仅在终态快照到达时消除已发送前缀。 StreamDeltaNormalizer deltaNormalizer = new StreamDeltaNormalizer(); // 取消输出标记。 AtomicBoolean cancelled = new AtomicBoolean(false); @@ -998,12 +998,12 @@ public class AgentScopeReActRuntime implements AgentRuntime { /** * 将 AgentScope 可能输出的累计快照归一化为增量。 * - *

主线路 mapper 要尽量保持 AgentScope 原始顺序,但不同模型或底层适配器可能 - * 输出累计文本。该归一化器只修正同一 message/block 的文本增量,不触碰旁路事件。

+ *

当前运行时显式使用 {@code incremental(true)}。普通事件携带新增文本,必须原样保留; + * {@code last=true} 的终态事件才可能携带完整快照,此时只发送尚未输出的尾部。

*/ private static final class StreamDeltaNormalizer { - private final Map previousValues = new LinkedHashMap<>(); + private final Map emittedValues = new LinkedHashMap<>(); /** * 归一化流式事件。 @@ -1030,10 +1030,14 @@ public class AgentScopeReActRuntime implements AgentRuntime { return; } String key = streamKey(event, payloadKey); - String previousText = previousValues.get(key); - previousValues.put(key, currentText); - if (previousText != null && !previousText.isEmpty() && currentText.startsWith(previousText)) { - event.getPayload().put(payloadKey, currentText.substring(previousText.length())); + boolean last = Boolean.TRUE.equals(event.getPayload().get("last")); + if (!last) { + emittedValues.computeIfAbsent(key, ignored -> new StringBuilder()).append(currentText); + return; + } + StringBuilder emitted = emittedValues.remove(key); + if (emitted != null && currentText.startsWith(emitted.toString())) { + event.getPayload().put(payloadKey, currentText.substring(emitted.length())); } } diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java index c4543e4..62fc4e8 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java @@ -690,6 +690,49 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertTrue(sessionStore.exists("session-1")); } + /** + * 验证增量模式会保留 URL 中连续出现的相同字符。 + */ + @Test + public void shouldPreserveRepeatedIdenticalTextDeltas() { + String expectedUrl = "http://127.0.0.1:39000/easyflow/file.docx"; + AgentScopeReActRuntime runtime = runtimeWithStreamingModel(List.of( + ChatResponse.builder() + .id("url-response") + .content(List.of(TextBlock.builder().text("http://127.0.0.1:39").build())) + .build(), + ChatResponse.builder() + .id("url-response") + .content(List.of(TextBlock.builder().text("0").build())) + .build(), + ChatResponse.builder() + .id("url-response") + .content(List.of(TextBlock.builder().text("0").build())) + .build(), + ChatResponse.builder() + .id("url-response") + .content(List.of(TextBlock.builder().text("0/easyflow/file.docx").build())) + .finishReason("stop") + .build())); + runtime.init(initRequest()); + + List events = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "create file")) + .collectList() + .block(); + String streamedText = events.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.MESSAGE_DELTA) + .map(event -> String.valueOf(event.getPayload().getOrDefault("text", ""))) + .reduce("", String::concat); + + Assert.assertEquals(expectedUrl, streamedText); + AgentRuntimeEvent completed = events.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED) + .findFirst() + .orElseThrow(); + Assert.assertEquals(expectedUrl, completed.getPayload().get("text")); + } + @Test public void shouldAllowNextStreamAfterPreviousStreamCompleted() { AgentScopeReActRuntime runtime = fakeRuntime(); @@ -1370,6 +1413,27 @@ public class AgentScopeStatefulRuntimeTest { new AgentScopeMessageAdapter()); } + /** + * 创建单次模型调用返回多个增量响应的运行时。 + * + * @param responses 同一次模型调用中的响应增量 + * @return 测试运行时 + */ + private AgentScopeReActRuntime runtimeWithStreamingModel(List responses) { + AgentScopeModelFactory modelFactory = new AgentScopeModelFactory() { + @Override + public Model create(AgentModelSpec modelSpec, + com.easyagents.agent.runtime.model.AgentGenerationOptions generationOptions) { + return new StreamingScriptedModel( + modelSpec == null ? "fake-model" : modelSpec.getModelName(), + responses); + } + }; + return new AgentScopeReActRuntime(modelFactory, new AgentScopeToolAdapter(), + new AgentScopeKnowledgeAdapter(), new AgentScopeMemoryAdapter(), new AgentScopeSkillAdapter(), + new AgentScopeMessageAdapter()); + } + /** * 根据审批事件创建恢复请求。 * @@ -1419,6 +1483,51 @@ public class AgentScopeStatefulRuntimeTest { } } + /** + * 单次调用按顺序返回全部响应增量的测试模型。 + */ + private static class StreamingScriptedModel implements Model { + + private final String modelName; + private final List responses; + + /** + * 创建流式测试模型。 + * + * @param modelName 模型名称 + * @param responses 响应增量 + */ + private StreamingScriptedModel(String modelName, List responses) { + this.modelName = modelName; + this.responses = responses; + } + + /** + * 返回预设的响应增量。 + * + * @param messages 输入消息 + * @param toolSchemas 工具定义 + * @param options 生成配置 + * @return 响应流 + */ + @Override + public Flux stream(List messages, + List toolSchemas, + GenerateOptions options) { + return Flux.fromIterable(responses); + } + + /** + * 获取模型名称。 + * + * @return 模型名称 + */ + @Override + public String getModelName() { + return modelName; + } + } + private AgentInitRequest initRequest() { AgentModelSpec modelSpec = new AgentModelSpec(); modelSpec.setModelName("fake-model"); From e995088d798540793fcf87867d0032b1672154c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 23 Jul 2026 20:19:23 +0800 Subject: [PATCH 08/33] =?UTF-8?q?build:=20=E4=BC=98=E5=8C=96=E5=A4=9A?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E5=8F=91=E5=B8=83=E6=89=93=E5=8C=85=E6=B5=81?= =?UTF-8?q?=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 升级源码、Javadoc 与 Central 发布插件并消除链接误报 - 将发布附件收敛到 release profile 并更新构建说明 --- README.md | 8 +++- pom.xml | 110 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 70ca387..35968ab 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,17 @@ Easy-Agents 是一个轻量、可扩展的 Java AI 应用开发框架,覆盖 在项目根目录执行: ```bash -mvn -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true clean install +mvn -DskipTests clean install ``` 构建完成后,相关构件会安装到本地 Maven 仓库,可供 `easyflow` 等项目直接依赖。 +发布时启用 `release` profile,生成源码包和 Javadoc 包,并调用 Maven Central 发布插件: + +```bash +mvn -Prelease -DskipTests deploy +``` + ## 快速示例 ```java diff --git a/pom.xml b/pom.xml index 446d648..28b5090 100644 --- a/pom.xml +++ b/pom.xml @@ -501,60 +501,67 @@ - - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - package - - jar-no-fork - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.4 - - EasyAgents - EasyAgents - private - false - true - true - -Xdoclint:none - true - 17 - - - - attach-javadocs - - jar - - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.7.0 - true - - central - true - - - + + + release + + + + org.apache.maven.plugins + maven-source-plugin + 3.4.0 + + + attach-sources + package + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.12.0 + + EasyAgents + EasyAgents + private + false + false + true + none + true + ${maven.compiler.release} + + + + attach-javadocs + package + + jar + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.11.0 + true + + central + true + + + + + + @@ -564,6 +571,5 @@ - From 6fa93bd6718c84c7d721f51f71c9a5468ee2a5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 27 Jul 2026 15:41:31 +0800 Subject: [PATCH 09/33] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20system=20=E6=B6=88=E6=81=AF=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 为 OpenAI 兼容模型增加字符串与文本数组两种 system content 策略 - 在 AgentScope 模型工厂统一安装兼容 formatter 并补充测试 --- .../agentscope/AgentOpenAIChatFormatter.java | 31 ++++++++++ .../agentscope/AgentScopeModelFactory.java | 4 ++ .../agent/runtime/model/AgentModelSpec.java | 21 +++++++ .../model/AgentSystemContentFormat.java | 13 ++++ .../AgentOpenAIChatFormatterTest.java | 62 +++++++++++++++++++ 5 files changed, 131 insertions(+) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java new file mode 100644 index 0000000..8f56602 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java @@ -0,0 +1,31 @@ +package com.easyagents.agent.runtime.agentscope; + +import io.agentscope.core.formatter.openai.OpenAIChatFormatter; +import io.agentscope.core.formatter.openai.dto.OpenAIContentPart; +import io.agentscope.core.formatter.openai.dto.OpenAIMessage; +import io.agentscope.core.message.Msg; + +import java.util.List; + +/** + * 为 OpenAI-compatible 模型补充 system content 内容块数组兼容能力。 + */ +public final class AgentOpenAIChatFormatter extends OpenAIChatFormatter { + + /** + * 将 AgentScope 消息转换为 OpenAI 消息,并规范 system 文本的 content 格式。 + * + * @param messages AgentScope 消息 + * @return OpenAI 请求消息 + */ + @Override + protected List doFormat(List messages) { + List formattedMessages = super.doFormat(messages); + for (OpenAIMessage message : formattedMessages) { + if ("system".equals(message.getRole()) && message.getContent() instanceof String text) { + message.setContent(List.of(OpenAIContentPart.text(text))); + } + } + return formattedMessages; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java index 15437d0..2f5ccdb 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java @@ -6,6 +6,7 @@ import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; import com.easyagents.agent.runtime.model.AgentModelFactory; import com.easyagents.agent.runtime.model.AgentModelProviderType; import com.easyagents.agent.runtime.model.AgentModelSpec; +import com.easyagents.agent.runtime.model.AgentSystemContentFormat; import io.agentscope.core.formatter.openai.DeepSeekFormatter; import io.agentscope.core.formatter.openai.GLMFormatter; import io.agentscope.core.model.*; @@ -153,6 +154,9 @@ public class AgentScopeModelFactory implements AgentModelFactory { .stream(Boolean.TRUE.equals(options.getStream())) .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) .generateOptions(options); + if (modelSpec.getSystemContentFormat() == AgentSystemContentFormat.TEXT_PARTS) { + builder.formatter(new AgentOpenAIChatFormatter()); + } return builder.build(); } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java index 59bc84a..ac33ff1 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java @@ -16,6 +16,7 @@ public class AgentModelSpec { private boolean supportImage; private boolean supportImageBase64Only; private AgentHttpVersionPolicy httpVersionPolicy = AgentHttpVersionPolicy.AUTO; + private AgentSystemContentFormat systemContentFormat = AgentSystemContentFormat.STRING; private Map metadata = new LinkedHashMap<>(); /** @@ -162,6 +163,26 @@ public class AgentModelSpec { this.httpVersionPolicy = httpVersionPolicy == null ? AgentHttpVersionPolicy.AUTO : httpVersionPolicy; } + /** + * 获取 OpenAI-compatible 请求中的 system content 格式。 + * + * @return system content 格式 + */ + public AgentSystemContentFormat getSystemContentFormat() { + return systemContentFormat; + } + + /** + * 设置 OpenAI-compatible 请求中的 system content 格式。 + * + * @param systemContentFormat system content 格式 + */ + public void setSystemContentFormat(AgentSystemContentFormat systemContentFormat) { + this.systemContentFormat = systemContentFormat == null + ? AgentSystemContentFormat.STRING + : systemContentFormat; + } + /** * 获取元数据。 * diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java new file mode 100644 index 0000000..f06fd45 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java @@ -0,0 +1,13 @@ +package com.easyagents.agent.runtime.model; + +/** + * OpenAI-compatible 请求中 system 消息的 content 格式。 + */ +public enum AgentSystemContentFormat { + + /** 使用字符串 content,保持 OpenAI Chat Completions 的常规格式。 */ + STRING, + + /** 使用仅包含 text 内容块的数组 content。 */ + TEXT_PARTS +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java new file mode 100644 index 0000000..3e4816e --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java @@ -0,0 +1,62 @@ +package com.easyagents.agent.runtime.agentscope; + +import com.easyagents.agent.runtime.model.AgentGenerationOptions; +import com.easyagents.agent.runtime.model.AgentModelProviderType; +import com.easyagents.agent.runtime.model.AgentModelSpec; +import com.easyagents.agent.runtime.model.AgentSystemContentFormat; +import io.agentscope.core.formatter.openai.dto.OpenAIContentPart; +import io.agentscope.core.formatter.openai.dto.OpenAIMessage; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.model.OpenAIChatModel; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.List; + +/** + * Agent OpenAI Chat 消息格式兼容测试。 + */ +public class AgentOpenAIChatFormatterTest { + + /** + * 验证 system 文本转换为 text 内容块数组,普通 user 文本保持字符串。 + */ + @Test + public void shouldConvertOnlySystemTextToContentParts() { + AgentOpenAIChatFormatter formatter = new AgentOpenAIChatFormatter(); + List messages = formatter.format(List.of( + Msg.builder().role(MsgRole.SYSTEM).textContent("system prompt").build(), + Msg.builder().role(MsgRole.USER).textContent("hello").build())); + + Assert.assertTrue(messages.get(0).getContent() instanceof List); + List contentParts = messages.get(0).getContentAsList(); + Assert.assertEquals(1, contentParts.size()); + Assert.assertEquals("text", contentParts.get(0).getType()); + Assert.assertEquals("system prompt", contentParts.get(0).getText()); + Assert.assertEquals("hello", messages.get(1).getContent()); + } + + /** + * 验证内容块数组策略会安装 EasyAgents 的 OpenAI Formatter。 + * + * @throws Exception 反射读取 Formatter 失败时抛出 + */ + @Test + public void textPartsPolicyShouldInstallAgentOpenAIFormatter() throws Exception { + AgentModelSpec spec = new AgentModelSpec(); + spec.setProviderType(AgentModelProviderType.OPENAI_COMPATIBLE); + spec.setModelName("vlm-test"); + spec.setBaseUrl("http://model.example.com/v1"); + spec.setApiKey("test-key"); + spec.setSystemContentFormat(AgentSystemContentFormat.TEXT_PARTS); + + OpenAIChatModel model = (OpenAIChatModel) new AgentScopeModelFactory() + .create(spec, new AgentGenerationOptions()); + Field formatterField = OpenAIChatModel.class.getDeclaredField("formatter"); + formatterField.setAccessible(true); + + Assert.assertTrue(formatterField.get(model) instanceof AgentOpenAIChatFormatter); + } +} From c48d9a9da637cffcca6e443a9b8d4c7c4cd30d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 27 Jul 2026 18:53:24 +0800 Subject: [PATCH 10/33] =?UTF-8?q?feat:=20=E5=AE=8C=E5=96=84=E6=A0=87?= =?UTF-8?q?=E5=87=86=20Skill=20=E5=8C=85=E5=BA=95=E5=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 标准化 SKILL.md、资源模型、校验规则与安全限额 - 支持流式内容存储和单、多 Skill ZIP 双向编解码 --- README.md | 1 + easy-agents-skill/README.md | 72 + easy-agents-skill/pom.xml | 8 + .../skill/codec/SkillPackageCodec.java | 45 +- .../skill/codec/SkillPackageReadMode.java | 18 + .../skill/codec/SkillPackageReadOptions.java | 69 + .../skill/codec/SkillPackageReadResult.java | 43 + .../skill/codec/SkillPackageWriteOptions.java | 38 + .../skill/codec/SkillPackageWriteResult.java | 61 + .../skill/codec/ZipSkillPackageCodec.java | 1709 +++++++++++++++-- .../exception/SkillPackageException.java | 64 +- .../exception/SkillValidationException.java | 81 +- .../skill/factory/SkillFactory.java | 66 +- .../com/easyagents/skill/model/Skill.java | 89 +- .../easyagents/skill/model/SkillAsset.java | 5 +- .../easyagents/skill/model/SkillDocument.java | 207 ++ .../easyagents/skill/model/SkillMetadata.java | 65 +- .../easyagents/skill/model/SkillPackage.java | 71 + .../skill/model/SkillPackageLayout.java | 16 + .../skill/model/SkillPackageLimits.java | 274 +++ .../skill/model/SkillReference.java | 5 +- .../easyagents/skill/model/SkillResource.java | 175 ++ .../skill/model/SkillResourceKind.java | 22 + .../easyagents/skill/model/SkillScript.java | 5 +- .../skill/model/SkillSourceLocation.java | 40 + .../memory/InMemorySkillRepository.java | 40 +- .../skill/store/SkillContentStage.java | 76 + .../skill/store/SkillContentStore.java | 168 +- .../file/TemporaryFileSkillContentStore.java | 633 ++++++ .../memory/InMemorySkillContentStore.java | 156 +- .../skill/util/SkillFrontmatter.java | 322 +++- .../easyagents/skill/util/SkillHashes.java | 32 +- .../com/easyagents/skill/util/SkillPaths.java | 48 +- .../easyagents/skill/util/SkillResources.java | 187 ++ .../com/easyagents/skill/util/SkillUtf8.java | 56 + .../validation/SkillValidationIssue.java | 79 + .../skill/validation/SkillValidationMode.java | 17 + .../validation/SkillValidationReport.java | 89 + .../validation/SkillValidationSeverity.java | 16 + .../skill/validation/SkillValidator.java | 49 + .../defaults/DefaultSkillValidator.java | 570 +++++- .../skill/codec/ZipSkillPackageCodecTest.java | 1422 ++++++++++++-- .../skill/model/SkillMetadataTest.java | 50 + .../skill/model/SkillPackageLimitsTest.java | 30 + .../memory/InMemorySkillRepositoryTest.java | 35 + .../TemporaryFileSkillContentStoreTest.java | 275 +++ .../memory/InMemorySkillContentStoreTest.java | 37 + .../skill/util/SkillFrontmatterTest.java | 199 ++ .../easyagents/skill/util/SkillPathsTest.java | 35 + .../skill/util/SkillResourcesTest.java | 22 + .../validation/SkillValidationReportTest.java | 37 + .../defaults/DefaultSkillValidatorTest.java | 532 +++-- pom.xml | 14 + 53 files changed, 7809 insertions(+), 666 deletions(-) create mode 100644 easy-agents-skill/README.md create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadMode.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadOptions.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadResult.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteOptions.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDocument.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackage.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLimits.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResourceKind.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillSourceLocation.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillUtf8.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationIssue.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationMode.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationReport.java create mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationSeverity.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillMetadataTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillPackageLimitsTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillFrontmatterTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/validation/SkillValidationReportTest.java diff --git a/README.md b/README.md index 35968ab..be229fa 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Easy-Agents 是一个轻量、可扩展的 Java AI 应用开发框架,覆盖 - `easy-agents-search-engine`:检索引擎实现。 - `easy-agents-tool`:工具调用能力。 - `easy-agents-mcp`:MCP 集成。 +- `easy-agents-skill`:标准 Agent Skills 包模型、安全校验、资源存储与 ZIP 双向编解码。 - `easy-agents-flow`:流程编排核心引擎。 - `easy-agents-support`:Flow 与 Easy-Agents 适配模块。 - `easy-agents-spring-boot-starter`:Spring Boot 自动配置支持。 diff --git a/easy-agents-skill/README.md b/easy-agents-skill/README.md new file mode 100644 index 0000000..08f0cd8 --- /dev/null +++ b/easy-agents-skill/README.md @@ -0,0 +1,72 @@ +# Easy-Agents Skill + +`easy-agents-skill` 提供标准 Agent Skills 包的领域模型、安全校验、通用资源存储,以及 ZIP 双向编解码能力。模块只负责 Skill 定义和包处理,不执行 `scripts/`,也不绑定具体智能体 Runtime。 + +## 标准包结构 + +Codec 支持根目录单 Skill、单目录 Skill 和多目录 Skill 三种输入布局。标准输出使用 `name/SKILL.md`,并保留以下可移植资源: + +- `references/` +- `scripts/` +- `assets/` +- `examples/` +- 其他安全相对路径资源 + +`SKILL.md` 使用 YAML frontmatter 与 Markdown 正文。未知字段、嵌套 Map/List、布尔值和数字会保留;语义校验通过结构化 issue 返回路径、行列、错误码与修复建议。 + +## 推荐调用方式 + +无参 `ZipSkillPackageCodec` 使用实例级临时文件存储,适合一次性导入导出。它拥有临时目录,必须关闭: + +```java +try (ZipSkillPackageCodec codec = new ZipSkillPackageCodec()) { + SkillPackageReadResult result = codec.decode( + inputStream, + SkillPackageReadOptions.defaults()); + SkillPackage skillPackage = result.getSkillPackage(); +} +``` + +生产系统需要让二进制资源跨请求存活时,应注入持久化的 `SkillContentStore`。注入存储的生命周期由调用方负责,关闭 Codec 不会关闭外部存储: + +```java +ZipSkillPackageCodec codec = new ZipSkillPackageCodec(contentStore); +SkillPackageReadResult result = codec.decode(inputStream, readOptions); +codec.encode(result.getSkillPackage(), outputStream, writeOptions); +``` + +成功解码的二进制资源通过 `contentRef` 引用已提交内容。业务侧丢弃包或删除资源时,应按持久化策略调用 `release`;复制引用时调用 `retain`。`REPORT_ONLY` 模式会回滚暂存内容,只用于检查诊断,不应持久化其资源引用。 + +## 校验上下文 + +校验通过 `SkillValidationMode` 区分两个明确上下文: + +- `DRAFT_IMPORT`:ZIP 导入和兼容预检使用;历史下划线名称保留为 warning,允许先进入草稿修复。 +- `STANDARD`:正式新建、发布校验和标准 ZIP 导出使用;下划线名称等互操作问题作为 error。 + +`SkillFactory.createStrict`、`SkillFactory.createWithResourcesStrict`、`DefaultSkillValidator.validate` 与 `ZipSkillPackageCodec.encode` 执行 `STANDARD` 校验。为保持旧调用兼容,`SkillFactory.create` 仍可构建导入草稿,原有 `validateReport(skill)` 与 `validateReport(skill, limits)` 继续使用 `DRAFT_IMPORT`;新调用方需要显式上下文时使用三参数 `validateReport`。 + +## 安全边界 + +默认 Codec 对读写两端执行统一限制: + +- 严格 UTF-8 文本和 ZIP entry 名称 +- Zip Slip、符号链接、路径大小写/Unicode 冲突与层级冲突防护 +- entry 数量、路径长度/深度、单文件、总解压大小、压缩包大小和压缩比限制 +- CRC、声明大小与实际流量复核 +- 安全 YAML 构造、重复 key、alias、深度和 code point 限制 +- stage / commit / rollback,失败时清理暂存内容 + +限额通过 `SkillPackageLimits` 配置,并由 `SkillPackageReadOptions`、`SkillPackageWriteOptions` 传入单次操作。 + +## 从旧接口迁移 + +`importZip(InputStream)` 为兼容入口,现已废弃。新调用方应使用 `decode`,以获得: + +- 包布局 `SkillPackageLayout` +- 标准化 `SkillPackage` +- 包哈希 +- 聚合校验报告 +- `STRICT` 或 `REPORT_ONLY` 读取模式 + +写出统一使用 `encode`。自定义校验器是附加业务校验,不能替代 Codec 内置的标准安全校验。 diff --git a/easy-agents-skill/pom.xml b/easy-agents-skill/pom.xml index 8dbe3b6..ec7ff75 100644 --- a/easy-agents-skill/pom.xml +++ b/easy-agents-skill/pom.xml @@ -18,6 +18,14 @@
+ + org.yaml + snakeyaml + + + org.apache.commons + commons-compress + junit junit diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java index 7936b0e..6da742d 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java @@ -1,20 +1,59 @@ package com.easyagents.skill.codec; +import com.easyagents.skill.exception.SkillPackageException; import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillPackage; +import com.easyagents.skill.model.SkillPackageLayout; +import com.easyagents.skill.validation.SkillValidationReport; import java.io.InputStream; +import java.io.OutputStream; import java.util.List; /** - * Skill 包导入接口。 + * Skill 包双向流式编解码接口。 */ public interface SkillPackageCodec { /** - * 从 zip 输入流导入 Skill。 + * 从 ZIP 输入流导入 Skill。 * - * @param inputStream zip 输入流 + * @param inputStream ZIP 输入流 * @return Skill 列表 + * @throws SkillPackageException ZIP 结构、内容或安全校验失败 + * @deprecated 请使用 {@link #decode(InputStream, SkillPackageReadOptions)} 获取包形态、hash 和诊断。 */ + @Deprecated List importZip(InputStream inputStream); + + /** + * 解码 Skill ZIP。 + * + *

兼容默认实现委托旧导入接口;正式 Codec 应覆盖。

+ * + * @param inputStream ZIP 输入流 + * @param options 读取选项 + * @return 解码结果 + * @throws SkillPackageException ZIP 结构、内容、安全校验或资源存储失败 + */ + default SkillPackageReadResult decode(InputStream inputStream, SkillPackageReadOptions options) { + List skills = importZip(inputStream); + SkillPackageLayout layout = skills.size() > 1 + ? SkillPackageLayout.MULTI_DIRECTORY : SkillPackageLayout.SINGLE_DIRECTORY; + return new SkillPackageReadResult(new SkillPackage(layout, skills), new SkillValidationReport(), null); + } + + /** + * 将 Skill 包编码为标准 ZIP。 + * + * @param skillPackage Skill 包 + * @param outputStream 输出流,不由本方法关闭 + * @param options 写出选项 + * @return 编码结果 + * @throws SkillPackageException Skill 包不合法、资源不可读或 ZIP 写出失败 + */ + default SkillPackageWriteResult encode(SkillPackage skillPackage, OutputStream outputStream, + SkillPackageWriteOptions options) { + throw new SkillPackageException("This SkillPackageCodec does not support encoding."); + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadMode.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadMode.java new file mode 100644 index 0000000..d1cb073 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadMode.java @@ -0,0 +1,18 @@ +package com.easyagents.skill.codec; + +/** + * Skill 包读取后的内容处理模式。 + */ +public enum SkillPackageReadMode { + + /** + * 校验通过后提交二进制内容;校验失败时抛出异常并回滚暂存内容。 + */ + COMMIT_ON_VALID, + + /** + * 返回可解析包的完整校验报告并回滚全部暂存内容,不提交二进制内容。 + * 返回资源中的 contentRef 仅表示内容哈希身份,不保证读取结果返回后仍可打开。 + */ + REPORT_ONLY +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadOptions.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadOptions.java new file mode 100644 index 0000000..984e3e2 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadOptions.java @@ -0,0 +1,69 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.model.SkillPackageLimits; + +/** + * Skill 包读取选项。 + */ +public final class SkillPackageReadOptions { + + private final SkillPackageLimits limits; + private final SkillPackageReadMode mode; + + /** + * 使用指定安全限额创建读取选项。 + * + * @param limits 安全限额 + */ + public SkillPackageReadOptions(SkillPackageLimits limits) { + this(limits, SkillPackageReadMode.COMMIT_ON_VALID); + } + + /** + * 使用指定安全限额和内容处理模式创建读取选项。 + * + * @param limits 安全限额 + * @param mode 内容处理模式 + */ + public SkillPackageReadOptions(SkillPackageLimits limits, SkillPackageReadMode mode) { + this.limits = limits == null ? SkillPackageLimits.defaults() : limits; + this.mode = mode == null ? SkillPackageReadMode.COMMIT_ON_VALID : mode; + } + + /** + * 创建默认读取选项。 + * + * @return 默认读取选项 + */ + public static SkillPackageReadOptions defaults() { + return new SkillPackageReadOptions(SkillPackageLimits.defaults()); + } + + /** + * 创建只返回校验报告且不提交内容的读取选项。 + * 二进制资源的 contentRef 仅表示内容哈希身份,不承诺可通过内容存储打开。 + * + * @return 只读预检选项 + */ + public static SkillPackageReadOptions reportOnly() { + return new SkillPackageReadOptions(SkillPackageLimits.defaults(), SkillPackageReadMode.REPORT_ONLY); + } + + /** + * 获取安全限额。 + * + * @return 安全限额 + */ + public SkillPackageLimits getLimits() { + return limits; + } + + /** + * 获取内容处理模式。 + * + * @return 内容处理模式 + */ + public SkillPackageReadMode getMode() { + return mode; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadResult.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadResult.java new file mode 100644 index 0000000..8a74b0f --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageReadResult.java @@ -0,0 +1,43 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.model.SkillPackage; +import com.easyagents.skill.validation.SkillValidationReport; + +/** + * Skill 包解码结果。 + */ +public final class SkillPackageReadResult { + + private final SkillPackage skillPackage; + private final SkillValidationReport validationReport; + private final String packageHash; + + /** + * 创建解码结果。 + * + * @param skillPackage Skill 包 + * @param validationReport 校验报告 + * @param packageHash 输入 ZIP SHA-256 + */ + public SkillPackageReadResult(SkillPackage skillPackage, SkillValidationReport validationReport, + String packageHash) { + this.skillPackage = skillPackage; + this.validationReport = validationReport; + this.packageHash = packageHash; + } + + /** @return Skill 包 */ + public SkillPackage getSkillPackage() { + return skillPackage; + } + + /** @return 结构化校验报告 */ + public SkillValidationReport getValidationReport() { + return validationReport; + } + + /** @return 输入 ZIP SHA-256 */ + public String getPackageHash() { + return packageHash; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteOptions.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteOptions.java new file mode 100644 index 0000000..900a711 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteOptions.java @@ -0,0 +1,38 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.model.SkillPackageLimits; + +/** + * Skill 包写出选项。 + */ +public final class SkillPackageWriteOptions { + + private final SkillPackageLimits limits; + + /** + * 使用指定安全限额创建写出选项。 + * + * @param limits 安全限额 + */ + public SkillPackageWriteOptions(SkillPackageLimits limits) { + this.limits = limits == null ? SkillPackageLimits.defaults() : limits; + } + + /** + * 创建默认写出选项。 + * + * @return 默认写出选项 + */ + public static SkillPackageWriteOptions defaults() { + return new SkillPackageWriteOptions(SkillPackageLimits.defaults()); + } + + /** + * 获取安全限额。 + * + * @return 安全限额 + */ + public SkillPackageLimits getLimits() { + return limits; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java new file mode 100644 index 0000000..35e0b96 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java @@ -0,0 +1,61 @@ +package com.easyagents.skill.codec; + +import com.easyagents.skill.model.SkillPackageLayout; + +/** + * Skill 包编码结果。 + */ +public final class SkillPackageWriteResult { + + private final String packageHash; + private final long size; + private final int entryCount; + private final SkillPackageLayout layout; + + /** + * 创建编码结果。 + * + * @param packageHash 输出 ZIP SHA-256 + * @param size 输出字节数 + * @param entryCount 输出 entry 数 + */ + public SkillPackageWriteResult(String packageHash, long size, int entryCount) { + this(packageHash, size, entryCount, SkillPackageLayout.SINGLE_DIRECTORY); + } + + /** + * 创建带包形态的编码结果。 + * + * @param packageHash 输出 ZIP SHA-256 + * @param size 输出字节数 + * @param entryCount 输出 entry 数 + * @param layout 输出包形态 + */ + public SkillPackageWriteResult(String packageHash, long size, int entryCount, + SkillPackageLayout layout) { + this.packageHash = packageHash; + this.size = size; + this.entryCount = entryCount; + this.layout = layout == null ? SkillPackageLayout.SINGLE_DIRECTORY : layout; + } + + /** @return 输出 ZIP SHA-256 */ + public String getPackageHash() { + return packageHash; + } + + /** @return 输出字节数 */ + public long getSize() { + return size; + } + + /** @return 输出 entry 数 */ + public int getEntryCount() { + return entryCount; + } + + /** @return 输出包形态 */ + public SkillPackageLayout getLayout() { + return layout; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java index 16590af..e7abef7 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java @@ -2,253 +2,1640 @@ package com.easyagents.skill.codec; import com.easyagents.skill.exception.SkillPackageException; import com.easyagents.skill.exception.SkillValidationException; -import com.easyagents.skill.factory.SkillFactory; -import com.easyagents.skill.model.*; +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillDocument; +import com.easyagents.skill.model.SkillMetadata; +import com.easyagents.skill.model.SkillPackage; +import com.easyagents.skill.model.SkillPackageLayout; +import com.easyagents.skill.model.SkillPackageLimits; +import com.easyagents.skill.model.SkillResource; +import com.easyagents.skill.model.SkillResourceKind; +import com.easyagents.skill.store.SkillContentStage; import com.easyagents.skill.store.SkillContentStore; -import com.easyagents.skill.store.memory.InMemorySkillContentStore; +import com.easyagents.skill.store.file.TemporaryFileSkillContentStore; +import com.easyagents.skill.util.SkillFrontmatter; import com.easyagents.skill.util.SkillHashes; import com.easyagents.skill.util.SkillPaths; +import com.easyagents.skill.util.SkillResources; +import com.easyagents.skill.util.SkillUtf8; +import com.easyagents.skill.validation.SkillValidationIssue; +import com.easyagents.skill.validation.SkillValidationMode; +import com.easyagents.skill.validation.SkillValidationReport; +import com.easyagents.skill.validation.SkillValidationSeverity; import com.easyagents.skill.validation.SkillValidator; import com.easyagents.skill.validation.defaults.DefaultSkillValidator; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.channels.SeekableByteChannel; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; +import java.util.zip.ZipOutputStream; /** - * 基于 zip 的 Skill 包导入实现。 + * 标准 Skill ZIP 的安全双向流式 Codec。 */ -public class ZipSkillPackageCodec implements SkillPackageCodec { +public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { private static final String DEFAULT_MEDIA_TYPE = "application/octet-stream"; + private static final int CENTRAL_DIRECTORY_HEADER_SIZE = 46; + private static final int CENTRAL_DIRECTORY_SIGNATURE = 0x02014B50; + private static final int END_OF_CENTRAL_DIRECTORY_MAX_SIZE = 65_557; + private static final int END_OF_CENTRAL_DIRECTORY_MIN_SIZE = 22; + private static final int END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054B50; + private static final long STABLE_ZIP_TIMESTAMP = 0L; + private static final int ZIP64_END_OF_CENTRAL_DIRECTORY_MIN_SIZE = 56; + private static final int ZIP64_END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06064B50; + private static final int ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIZE = 20; + private static final int ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE = 0x07064B50; + private static final long UINT32_MAX = 0xFFFF_FFFFL; + private static final int UINT16_MAX = 0xFFFF; private final SkillContentStore contentStore; - private final SkillValidator validator; + private final SkillValidator additionalValidator; + private final boolean ownsContentStore; /** - * 创建使用内存内容存储的 zip Skill 包导入器。 + * 创建使用临时文件内容存储的 ZIP Codec。 + * + *

该实例拥有默认内容存储,使用完毕后应调用 {@link #close()} 清理临时内容。

*/ public ZipSkillPackageCodec() { - this(new InMemorySkillContentStore()); + this(new TemporaryFileSkillContentStore(), null, true); } /** - * 创建 zip Skill 包导入器。 + * 创建 ZIP Codec。 * * @param contentStore 二进制内容存储 */ public ZipSkillPackageCodec(SkillContentStore contentStore) { - this(contentStore, new DefaultSkillValidator()); + this(contentStore, null, false); } /** - * 创建 zip Skill 包导入器。 + * 创建 ZIP Codec。 * * @param contentStore 二进制内容存储 * @param validator Skill 聚合校验器 */ public ZipSkillPackageCodec(SkillContentStore contentStore, SkillValidator validator) { + this(contentStore, requireValidator(validator), false); + } + + private ZipSkillPackageCodec(SkillContentStore contentStore, SkillValidator validator, + boolean ownsContentStore) { if (contentStore == null) { throw new SkillPackageException("Skill content store is required."); } + this.contentStore = contentStore; + this.additionalValidator = validator; + this.ownsContentStore = ownsContentStore; + } + + private static SkillValidator requireValidator(SkillValidator validator) { if (validator == null) { throw new SkillPackageException("Skill validator is required."); } - this.contentStore = contentStore; - this.validator = validator; + return validator; } /** - * 从 zip 流导入 Skill 列表。 + * 关闭 Codec 自有的默认临时内容存储。 * - * @param inputStream zip 输入流 - * @return Skill 列表 + *

通过构造函数注入的内容存储由调用方管理,本方法不会关闭它。

*/ @Override + public void close() { + if (ownsContentStore) { + ((TemporaryFileSkillContentStore) contentStore).close(); + } + } + + /** + * 使用兼容入口导入 ZIP。 + * + * @param inputStream ZIP 输入流 + * @return Skill 列表 + * @deprecated 请使用 {@link #decode(InputStream, SkillPackageReadOptions)}。 + */ + @Deprecated + @Override public List importZip(InputStream inputStream) { + return decode(inputStream, SkillPackageReadOptions.defaults()).getSkillPackage().getSkills(); + } + + /** + * 安全解码 Skill ZIP。 + * + * @param inputStream ZIP 输入流 + * @param options 读取选项 + * @return 解码结果 + */ + @Override + public SkillPackageReadResult decode(InputStream inputStream, SkillPackageReadOptions options) { if (inputStream == null) { - throw new SkillPackageException("Zip input stream is required."); + throw packageError("ZIP_INPUT_REQUIRED", null, "ZIP input stream is required."); } - Map builders = new LinkedHashMap<>(); - try (ZipInputStream zipInputStream = new ZipInputStream(inputStream)) { - ZipEntry entry; - while ((entry = zipInputStream.getNextEntry()) != null) { - handleEntry(entry, zipInputStream, builders); - zipInputStream.closeEntry(); + SkillPackageReadOptions effectiveOptions = options == null + ? SkillPackageReadOptions.defaults() : options; + SkillPackageLimits limits = effectiveOptions.getLimits(); + Path temporaryZip = null; + StageTracker stages = new StageTracker(contentStore); + try { + temporaryZip = Files.createTempFile("easy-agents-skill-", ".zip"); + InputCopyResult copyResult = copyInput(inputStream, temporaryZip, + limits.getMaxCompressedPackageBytes()); + validateZipSignature(temporaryZip); + DecodeState state = readArchive(temporaryZip, limits, stages); + SkillValidationReport report = new SkillValidationReport(); + for (Skill skill : state.skills) { + String packageRoot = state.layout == SkillPackageLayout.ROOT_SKILL + ? null : skill.getPackageRoot(); + report.merge(prefixValidationPaths( + validateForCodec(skill, limits, SkillValidationMode.DRAFT_IMPORT), + packageRoot)); } + if (effectiveOptions.getMode() == SkillPackageReadMode.REPORT_ONLY) { + stages.rollbackAll(); + return new SkillPackageReadResult( + new SkillPackage(state.layout, state.skills), report, copyResult.hash); + } + if (report.hasErrors()) { + throw new SkillPackageException("Skill package validation failed.", report); + } + stages.commitAll(); + return new SkillPackageReadResult( + new SkillPackage(state.layout, state.skills), report, copyResult.hash); + } catch (SkillPackageException e) { + stages.cleanupAfterFailure(e); + throw e; + } catch (SkillValidationException e) { + stages.cleanupAfterFailure(e); + throw validationPackageError(e); } catch (IOException e) { - throw new SkillPackageException("Failed to import skill zip package.", e); + stages.cleanupAfterFailure(e); + throw new SkillPackageException("ZIP_IO_ERROR", null, + "Failed to read Skill ZIP package.", e); + } catch (RuntimeException e) { + stages.cleanupAfterFailure(e); + throw new SkillPackageException("SKILL_CONTENT_STORE_ERROR", null, + "Failed to persist Skill package content.", e); + } finally { + deleteQuietly(temporaryZip); } - if (builders.isEmpty()) { - throw new SkillPackageException("Zip package must contain at least one skill folder."); - } - List skills = new ArrayList<>(); - for (SkillBuilder builder : builders.values()) { - Skill skill = builder.build(); - validator.validate(skill); - skills.add(skill); - } - for (SkillBuilder builder : builders.values()) { - builder.writeAssets(contentStore); - } - return skills; } - private void handleEntry(ZipEntry entry, ZipInputStream zipInputStream, Map builders) + /** + * 将 Skill 包稳定编码为标准 ZIP。 + * + * @param skillPackage Skill 包 + * @param outputStream 输出流,不由本方法关闭 + * @param options 写出选项 + * @return 编码结果 + */ + @Override + public SkillPackageWriteResult encode(SkillPackage skillPackage, OutputStream outputStream, + SkillPackageWriteOptions options) { + if (skillPackage == null || skillPackage.getSkills() == null || skillPackage.getSkills().isEmpty()) { + throw packageError("EMPTY_SKILL_PACKAGE", null, "Skill package must contain at least one Skill."); + } + if (outputStream == null) { + throw packageError("ZIP_OUTPUT_REQUIRED", null, "ZIP output stream is required."); + } + SkillPackageWriteOptions effectiveOptions = options == null + ? SkillPackageWriteOptions.defaults() : options; + SkillPackageLimits limits = effectiveOptions.getLimits(); + List files = prepareOutput(skillPackage.getSkills(), limits); + Path temporaryZip = null; + try { + temporaryZip = Files.createTempFile("easy-agents-skill-export-", ".zip"); + OutputWriteResult writeResult = writeArchive(temporaryZip, files, limits); + try (InputStream input = Files.newInputStream(temporaryZip)) { + transfer(input, outputStream, limits.getMaxCompressedPackageBytes(), null); + } + outputStream.flush(); + SkillPackageLayout outputLayout = skillPackage.getSkills().size() > 1 + ? SkillPackageLayout.MULTI_DIRECTORY : SkillPackageLayout.SINGLE_DIRECTORY; + return new SkillPackageWriteResult( + writeResult.hash, writeResult.size, files.size(), outputLayout); + } catch (SkillPackageException e) { + throw e; + } catch (IOException e) { + if (hasCause(e, CompressedSizeLimitException.class)) { + throw packageError("COMPRESSED_SIZE_LIMIT", null, + "Compressed Skill ZIP exceeds " + limits.getMaxCompressedPackageBytes() + " bytes."); + } + throw new SkillPackageException("ZIP_IO_ERROR", null, + "Failed to write Skill ZIP package.", e); + } finally { + deleteQuietly(temporaryZip); + } + } + + private DecodeState readArchive(Path archivePath, SkillPackageLimits limits, StageTracker stages) throws IOException { - String rawPath = entry.getName(); - if (entry.isDirectory() || SkillPaths.isIgnoredSystemPath(rawPath)) { - return; - } - String normalizedPath = SkillPaths.normalize(rawPath); - if (SkillPaths.SKILL_FILE.equals(normalizedPath)) { - throw new SkillPackageException("Zip root cannot directly contain SKILL.md."); - } - int slashIndex = normalizedPath.indexOf('/'); - if (slashIndex < 0) { - throw new SkillPackageException("Zip root can only contain skill folders: " + normalizedPath); - } + enforceCentralDirectoryEntryLimit(archivePath, limits.getMaxEntryCount()); + List files = new ArrayList<>(); + Set exactPaths = new HashSet<>(); + Map collisionPaths = new HashMap<>(); + Map descendantPaths = new HashMap<>(); + long declaredTotal = 0; + int entryCount = 0; - String skillId = normalizedPath.substring(0, slashIndex); - String skillPath = normalizedPath.substring(slashIndex + 1); - if (skillPath.isBlank()) { - return; + try (ZipFile zipFile = new ZipFile(archivePath)) { + Enumeration entries = zipFile.getEntriesInPhysicalOrder(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + String rawPath = strictUtf8EntryName(entry); + entryCount++; + if (entryCount > limits.getMaxEntryCount()) { + throw packageError("ENTRY_COUNT_LIMIT", rawPath, + "Skill ZIP exceeds " + limits.getMaxEntryCount() + " entries."); + } + if (entry.isUnixSymlink()) { + throw packageError("SYMLINK_ENTRY", rawPath, + "Symbolic link entries are not allowed in Skill packages."); + } + if (!zipFile.canReadEntryData(entry)) { + throw packageError("UNSUPPORTED_ZIP_ENTRY", rawPath, + "Encrypted or unsupported ZIP entries are not allowed."); + } + String pathForValidation = entry.isDirectory() ? stripDirectorySuffix(rawPath) : rawPath; + String path = normalizeArchivePath(pathForValidation, limits); + if (entry.isDirectory() || SkillPaths.isIgnoredSystemPath(rawPath)) { + continue; + } + if (!exactPaths.add(path)) { + throw packageError("DUPLICATE_ENTRY", path, "Duplicate ZIP entry: " + path); + } + String collisionKey = SkillPaths.collisionKey(path); + String conflictingPath = collisionPaths.get(collisionKey); + if (conflictingPath != null) { + throw packageError("PATH_COLLISION", path, + "ZIP entry conflicts with " + conflictingPath + " after Unicode/case normalization."); + } + String hierarchyConflict = findHierarchyConflict( + collisionKey, collisionPaths, descendantPaths); + if (hierarchyConflict != null) { + throw packageError("PATH_HIERARCHY_CONFLICT", path, + "ZIP entry conflicts with file " + hierarchyConflict + + " in the same path hierarchy."); + } + collisionPaths.put(collisionKey, path); + indexDescendantPath(collisionKey, path, descendantPaths); + + long size = entry.getSize(); + long compressedSize = entry.getCompressedSize(); + if (size < 0 || compressedSize < 0) { + throw packageError("UNKNOWN_ENTRY_SIZE", path, + "ZIP entry sizes must be present in the central directory."); + } + if (entry.getCrc() < 0) { + throw packageError("UNKNOWN_ENTRY_CRC", path, + "ZIP entry CRC must be present in the central directory."); + } + checkCompressionRatio(path, size, compressedSize, limits); + try { + declaredTotal = Math.addExact(declaredTotal, size); + } catch (ArithmeticException e) { + throw packageError("TOTAL_SIZE_LIMIT", path, "Skill ZIP uncompressed size overflow."); + } + if (declaredTotal > limits.getMaxTotalUncompressedBytes()) { + throw packageError("TOTAL_SIZE_LIMIT", path, + "Skill ZIP exceeds " + limits.getMaxTotalUncompressedBytes() + + " uncompressed bytes."); + } + files.add(new ArchiveFile(entry, path)); + } + if (files.isEmpty()) { + throw packageError("EMPTY_SKILL_PACKAGE", null, + "ZIP package must contain at least one Skill."); + } + + ArchiveLayout archiveLayout = determineLayout(files); + List skills = new ArrayList<>(); + for (ArchiveGroup group : archiveLayout.groups) { + skills.add(readSkill(zipFile, group, archiveLayout.layout, limits, stages)); + } + return new DecodeState(archiveLayout.layout, skills); + } catch (SkillPackageException e) { + throw e; + } catch (IOException e) { + throw new SkillPackageException("INVALID_ZIP", null, + "Skill package ZIP structure or entry data is invalid.", e); } - SkillBuilder builder = builders.computeIfAbsent(skillId, SkillBuilder::new); - builder.addFile(skillPath, readEntryBytes(zipInputStream)); } - private static byte[] readEntryBytes(InputStream inputStream) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int readLength; - while ((readLength = inputStream.read(buffer)) >= 0) { - outputStream.write(buffer, 0, readLength); + private Skill readSkill(ZipFile zipFile, ArchiveGroup group, SkillPackageLayout layout, + SkillPackageLimits limits, StageTracker stages) throws IOException { + ArchiveFile skillFile = group.files.stream() + .filter(file -> SkillPaths.SKILL_FILE.equals(file.relativePath)) + .findFirst() + .orElseThrow(() -> packageError("SKILL_FILE_REQUIRED", group.root, + "Skill directory must contain exactly one SKILL.md.")); + + String skillContent = readStrictText(zipFile, skillFile, limits.getMaxTextFileBytes()); + SkillDocument document; + try { + document = SkillFrontmatter.parseDocument(skillContent, limits); + } catch (SkillValidationException e) { + throw validationPackageError(e, + layout == SkillPackageLayout.ROOT_SKILL ? null : group.root); } - return outputStream.toByteArray(); + Map frontmatter = document.getFrontmatter().getValues(); + String name = scalar(frontmatter.get("name")); + String description = scalar(frontmatter.get("description")); + + List resources = new ArrayList<>(); + for (ArchiveFile file : group.files) { + if (SkillPaths.SKILL_FILE.equals(file.relativePath)) { + continue; + } + SkillResource resource = readResource(zipFile, file, limits, stages); + resources.add(resource); + } + resources.sort(Comparator.comparing(SkillResource::getPath)); + + Skill skill = new Skill(); + skill.setId(null); + skill.setPackageRoot(layout == SkillPackageLayout.ROOT_SKILL ? name : group.root); + skill.setName(name); + skill.setDescription(description); + skill.setMetadata(new SkillMetadata(frontmatter)); + skill.setDocument(document); + skill.setResources(resources); + SkillResources.refreshLegacyViews(skill); + return skill; } - private static String detectMediaType(String path) { - String mediaType = URLConnection.guessContentTypeFromName(path); - return mediaType == null ? DEFAULT_MEDIA_TYPE : mediaType; + private SkillResource readResource(ZipFile zipFile, ArchiveFile file, SkillPackageLimits limits, + StageTracker stages) throws IOException { + SkillResourceKind kind = SkillResources.classify(file.relativePath); + String mediaType = detectMediaType(file.relativePath); + boolean text = SkillResources.isText(file.relativePath, kind, mediaType); + long singleLimit = text ? limits.getMaxTextFileBytes() : limits.getMaxBinaryFileBytes(); + if (file.entry.getSize() > singleLimit) { + throw packageError(text ? "TEXT_FILE_SIZE_LIMIT" : "BINARY_FILE_SIZE_LIMIT", file.fullPath, + "Skill resource exceeds " + singleLimit + " bytes."); + } + + SkillResource resource = new SkillResource(); + resource.setPath(file.relativePath); + resource.setKind(kind); + resource.setMediaType(mediaType); + if (text) { + String content = readStrictText(zipFile, file, singleLimit); + byte[] bytes = strictUtf8Bytes(content, file.fullPath); + resource.setTextContent(content); + resource.setContentHash(SkillHashes.sha256Hex(bytes)); + resource.setSize(bytes.length); + } else { + CRC32 crc = new CRC32(); + try (InputStream input = zipFile.getInputStream(file.entry); + CheckedInputStream checkedInput = new CheckedInputStream(input, crc)) { + SkillContentStage stage = contentStore.stage(checkedInput, singleLimit); + if (stage.getSize() != file.entry.getSize()) { + contentStore.rollback(stage); + throw packageError("ENTRY_SIZE_MISMATCH", file.fullPath, + "ZIP entry size changed while reading."); + } + try { + verifyCrc(file.entry, crc.getValue(), file.fullPath); + } catch (RuntimeException e) { + contentStore.rollback(stage); + throw e; + } + resource.setContentRef(stage.getContentRef()); + resource.setContentHash(stage.getContentHash()); + resource.setSize(stage.getSize()); + stages.add(stage, resource, file.fullPath); + } + } + return resource; } - private static final class SkillBuilder { - - private final String id; - private final Set paths = new HashSet<>(); - private String skillContent; - private final List references = new ArrayList<>(); - private final List scripts = new ArrayList<>(); - private final List assets = new ArrayList<>(); - private final List pendingAssets = new ArrayList<>(); - - private SkillBuilder(String id) { - this.id = id; + private static ArchiveLayout determineLayout(List files) { + boolean hasRootSkill = files.stream().anyMatch(file -> SkillPaths.SKILL_FILE.equals(file.fullPath)); + if (hasRootSkill) { + boolean containsNestedSkill = files.stream().anyMatch(file -> + !SkillPaths.SKILL_FILE.equals(file.fullPath) + && file.fullPath.endsWith("/" + SkillPaths.SKILL_FILE)); + if (containsNestedSkill) { + throw packageError("MIXED_PACKAGE_LAYOUT", null, + "Root SKILL.md cannot be mixed with wrapped Skill directories."); + } + List relativeFiles = files.stream() + .map(file -> file.withRelativePath(file.fullPath)) + .toList(); + return new ArchiveLayout(SkillPackageLayout.ROOT_SKILL, + List.of(new ArchiveGroup(null, relativeFiles))); } - private void addFile(String path, byte[] bytes) { - String normalizedPath = SkillPaths.normalize(path); - if (!paths.add(normalizedPath)) { - throw new SkillPackageException("Duplicate skill file path: " + id + "/" + normalizedPath); - } - if (SkillPaths.SKILL_FILE.equals(normalizedPath)) { - addSkillFile(bytes); - return; - } - - String topDir = SkillPaths.firstSegment(normalizedPath); - if (SkillPaths.REFERENCES_DIR.equals(topDir)) { - addReference(normalizedPath, bytes); - } else if (SkillPaths.SCRIPTS_DIR.equals(topDir)) { - addScript(normalizedPath, bytes); - } else if (SkillPaths.ASSETS_DIR.equals(topDir)) { - addAsset(normalizedPath, bytes); - } else { - throw new SkillPackageException("Unknown skill top-level directory: " + id + "/" + normalizedPath); + Map> grouped = new LinkedHashMap<>(); + for (ArchiveFile file : files) { + int separator = file.fullPath.indexOf('/'); + if (separator < 1 || separator == file.fullPath.length() - 1) { + throw packageError("UNOWNED_ROOT_FILE", file.fullPath, + "Wrapped Skill ZIP root can contain only Skill directories."); } + String root = file.fullPath.substring(0, separator); + String relativePath = file.fullPath.substring(separator + 1); + grouped.computeIfAbsent(root, ignored -> new ArrayList<>()) + .add(file.withRelativePath(relativePath)); } - - private void addSkillFile(byte[] bytes) { - skillContent = new String(bytes, StandardCharsets.UTF_8); - } - - private void addReference(String path, byte[] bytes) { - if (!SkillPaths.hasExtension(path, ".md")) { - throw new SkillPackageException("Skill reference must be a markdown file: " + id + "/" + path); + List groups = new ArrayList<>(); + for (Map.Entry> entry : grouped.entrySet()) { + long skillFileCount = entry.getValue().stream() + .filter(file -> SkillPaths.SKILL_FILE.equals(file.relativePath)) + .count(); + if (skillFileCount != 1) { + throw packageError("SKILL_FILE_REQUIRED", entry.getKey(), + "Each top-level Skill directory must contain exactly one SKILL.md."); } - String content = new String(bytes, StandardCharsets.UTF_8); - SkillReference reference = new SkillReference(); - reference.setPath(path); - reference.setName(SkillPaths.fileName(path)); - reference.setContent(content); - reference.setContentHash(SkillHashes.sha256Hex(bytes)); - reference.setSize(bytes.length); - references.add(reference); + groups.add(new ArchiveGroup(entry.getKey(), entry.getValue())); } + groups.sort(Comparator.comparing(group -> group.root)); + SkillPackageLayout layout = groups.size() == 1 + ? SkillPackageLayout.SINGLE_DIRECTORY : SkillPackageLayout.MULTI_DIRECTORY; + return new ArchiveLayout(layout, groups); + } - private void addScript(String path, byte[] bytes) { - SkillScriptLanguage language = SkillScriptLanguage.fromPath(path); - if (language == SkillScriptLanguage.UNKNOWN) { - throw new SkillPackageException("Unsupported skill script extension: " + id + "/" + path); + /** + * 执行 Codec 不可绕过的标准安全校验,并追加调用方业务校验。 + * + * @param skill Skill 聚合 + * @param limits 当前读写操作限额 + * @param mode 标准校验模式 + * @return 合并后的结构化报告 + */ + private SkillValidationReport validateForCodec(Skill skill, SkillPackageLimits limits, + SkillValidationMode mode) { + SkillValidationReport report = new DefaultSkillValidator(limits) + .validateReport(skill, limits, mode); + if (additionalValidator != null) { + SkillValidationReport additionalReport = additionalValidator.getClass() == DefaultSkillValidator.class + ? additionalValidator.validateReport(skill, null, mode) + : additionalValidator.validateReport(skill, limits, mode); + report.merge(additionalReport); + } + return report; + } + + private List prepareOutput(List skills, SkillPackageLimits limits) { + SkillValidationReport report = new SkillValidationReport(); + Set roots = new HashSet<>(); + List files = new ArrayList<>(); + long totalSize = 0; + + for (Skill skill : skills) { + SkillValidationReport skillReport = validateForCodec( + skill, limits, SkillValidationMode.STANDARD); + String diagnosticRoot = null; + if (skills.size() > 1 && skill != null) { + diagnosticRoot = skill.getPackageRoot(); + if (diagnosticRoot == null || diagnosticRoot.isBlank()) { + diagnosticRoot = skill.getName(); + } } - SkillScript script = new SkillScript(); - script.setPath(path); - script.setLanguage(language); - script.setContent(new String(bytes, StandardCharsets.UTF_8)); - script.setContentHash(SkillHashes.sha256Hex(bytes)); - script.setSize(bytes.length); - scripts.add(script); - } - - private void addAsset(String path, byte[] bytes) { - String contentHash = SkillHashes.sha256Hex(bytes); - SkillAsset asset = new SkillAsset(); - asset.setPath(path); - asset.setName(SkillPaths.fileName(path)); - asset.setMediaType(detectMediaType(path)); - asset.setContentRef("sha256:" + contentHash); - asset.setContentHash(contentHash); - asset.setSize(bytes.length); - assets.add(asset); - pendingAssets.add(new PendingAsset(asset, bytes)); - } - - private Skill build() { - if (skillContent == null) { - throw new SkillPackageException("Skill folder must contain SKILL.md: " + id); + report.merge(prefixValidationPaths(skillReport, diagnosticRoot)); + if (skill == null || skillReport.hasErrors()) { + continue; } - try { - return SkillFactory.create(id, skillContent, references, scripts, assets); - } catch (SkillValidationException e) { - throw new SkillPackageException("Invalid SKILL.md frontmatter: " + id, e); + if (skill.getName() == null + || !skill.getName().matches("[a-z0-9]+(?:-[a-z0-9]+)*")) { + report.add(new SkillValidationIssue("NON_CANONICAL_EXPORT_NAME", + SkillValidationSeverity.ERROR, SkillPaths.SKILL_FILE, null, null, + "Standard export requires a canonical hyphenated Skill name.", + "Rename the Skill before export.")); + continue; + } + String collisionRoot = skill.getName().toLowerCase(Locale.ROOT); + if (!roots.add(collisionRoot)) { + report.add(new SkillValidationIssue("DUPLICATE_SKILL_NAME", SkillValidationSeverity.ERROR, + SkillPaths.SKILL_FILE, null, null, + "Skill package contains duplicate names: " + skill.getName(), null)); + continue; } - } - private void writeAssets(SkillContentStore contentStore) { - for (PendingAsset pendingAsset : pendingAssets) { - String contentRef = contentStore.put(pendingAsset.bytes); - if (!pendingAsset.asset.getContentRef().equals(contentRef)) { - throw new SkillPackageException("Skill asset content ref does not match store result: " - + id + "/" + pendingAsset.asset.getPath()); + String skillContent = skill.getSkillContent(); + String skillPath = skill.getName() + "/" + SkillPaths.SKILL_FILE; + validateOutputPath(skillPath, limits); + long skillSize = utf8Size(skillContent, skillPath); + if (skillSize > limits.getMaxTextFileBytes()) { + report.add(new SkillValidationIssue("TEXT_FILE_SIZE_LIMIT", SkillValidationSeverity.ERROR, + skillPath, null, null, + "SKILL.md exceeds the text file size limit.", null)); + } + files.add(OutputFile.text(skillPath, skillContent)); + totalSize = checkedOutputTotal(totalSize, skillSize, limits, skillPath); + + List resources = SkillResources.canonicalResources(skill); + resources.sort(Comparator.comparing(SkillResource::getPath)); + for (SkillResource resource : resources) { + String path = skill.getName() + "/" + SkillPaths.normalize(resource.getPath()); + validateOutputPath(path, limits); + if (resource.isText()) { + String textContent = resource.getTextContent(); + long textSize = utf8Size(textContent, path); + if (textSize > limits.getMaxTextFileBytes()) { + report.add(new SkillValidationIssue("TEXT_FILE_SIZE_LIMIT", + SkillValidationSeverity.ERROR, path, null, null, + "Text Skill resource exceeds the text file size limit.", null)); + continue; + } + files.add(OutputFile.text(path, textContent)); + totalSize = checkedOutputTotal(totalSize, textSize, limits, path); + } else { + if (resource.getSize() > limits.getMaxBinaryFileBytes()) { + report.add(new SkillValidationIssue("BINARY_FILE_SIZE_LIMIT", + SkillValidationSeverity.ERROR, path, null, null, + "Binary Skill resource exceeds the binary file size limit.", null)); + continue; + } + if (!contentExists(resource.getContentRef(), path)) { + report.add(new SkillValidationIssue("CONTENT_NOT_FOUND", SkillValidationSeverity.ERROR, + path, null, null, "Skill resource content does not exist.", null)); + continue; + } + files.add(OutputFile.binary(path, resource)); + totalSize = checkedOutputTotal(totalSize, resource.getSize(), limits, path); } } } + if (files.size() > limits.getMaxEntryCount()) { + report.add(new SkillValidationIssue("ENTRY_COUNT_LIMIT", SkillValidationSeverity.ERROR, + null, null, null, "Skill export exceeds the entry count limit.", null)); + } + if (report.hasErrors()) { + throw new SkillPackageException("Skill package cannot be exported.", report); + } + + files.sort(Comparator.comparing(file -> file.path)); + Set paths = new HashSet<>(); + Map collisionPaths = new HashMap<>(); + Map descendantPaths = new HashMap<>(); + for (OutputFile file : files) { + String collisionKey = SkillPaths.collisionKey(file.path); + String collision = collisionPaths.get(collisionKey); + if (!paths.add(file.path) || collision != null) { + throw packageError("PATH_COLLISION", file.path, + "Skill export contains duplicate or case-conflicting paths."); + } + String hierarchyConflict = findHierarchyConflict( + collisionKey, collisionPaths, descendantPaths); + if (hierarchyConflict != null) { + throw packageError("PATH_HIERARCHY_CONFLICT", file.path, + "Skill export path conflicts with file " + hierarchyConflict + + " in the same path hierarchy."); + } + collisionPaths.put(collisionKey, file.path); + indexDescendantPath(collisionKey, file.path, descendantPaths); + } + return files; } - private static final class PendingAsset { + private OutputWriteResult writeArchive(Path archivePath, List files, + SkillPackageLimits limits) throws IOException { + OutputWriteResult initialResult = writeArchiveAttempt(archivePath, files, limits, Map.of()); + Map storedEntries = findEntriesExceedingCompressionRatio( + archivePath, limits); + if (storedEntries.isEmpty()) { + return initialResult; + } - private final SkillAsset asset; - private final byte[] bytes; + OutputWriteResult rewrittenResult = writeArchiveAttempt( + archivePath, files, limits, storedEntries); + Map remainingViolations = findEntriesExceedingCompressionRatio( + archivePath, limits); + if (!remainingViolations.isEmpty()) { + String path = remainingViolations.keySet().iterator().next(); + throw packageError("COMPRESSION_RATIO_LIMIT", path, + "Encoded Skill ZIP cannot satisfy the configured compression ratio limit."); + } + return rewrittenResult; + } - private PendingAsset(SkillAsset asset, byte[] bytes) { - this.asset = asset; - this.bytes = bytes; + /** + * 按指定 entry 存储策略写出一次稳定 ZIP。 + * + * @param archivePath 临时归档路径 + * @param files 输出文件计划 + * @param limits 当前写出限额 + * @param storedEntries 需要使用 STORED 的 entry 元数据 + * @return 本次写出结果 + * @throws IOException 写出失败 + */ + private OutputWriteResult writeArchiveAttempt(Path archivePath, List files, + SkillPackageLimits limits, + Map storedEntries) + throws IOException { + MessageDigest digest = SkillHashes.newSha256Digest(); + LimitedOutputStream limitedOutput = new LimitedOutputStream( + Files.newOutputStream(archivePath, StandardOpenOption.TRUNCATE_EXISTING), + limits.getMaxCompressedPackageBytes()); + try (DigestOutputStream digestOutput = new DigestOutputStream(limitedOutput, digest); + ZipOutputStream zipOutput = new ZipOutputStream(digestOutput, StandardCharsets.UTF_8)) { + for (OutputFile file : files) { + ZipEntry entry = new ZipEntry(file.path); + entry.setTime(STABLE_ZIP_TIMESTAMP); + StoredEntryMetadata storedMetadata = storedEntries.get(file.path); + if (storedMetadata != null) { + entry.setMethod(ZipEntry.STORED); + entry.setSize(storedMetadata.size); + entry.setCompressedSize(storedMetadata.size); + entry.setCrc(storedMetadata.crc); + } + zipOutput.putNextEntry(entry); + if (file.text != null) { + writeTextResource(zipOutput, file.text, file.path); + } else { + writeBinaryResource(zipOutput, file.resource, limits, file.path); + } + zipOutput.closeEntry(); + } + zipOutput.finish(); + } + return new OutputWriteResult(SkillHashes.toHex(digest.digest()), limitedOutput.getCount()); + } + + /** + * 找出会在同一限额下被解码器判定为高压缩比的 entry。 + * + * @param archivePath 已完成的临时归档 + * @param limits 当前写出限额 + * @return 需要改用 STORED 的 entry 元数据 + * @throws IOException 归档检查失败 + */ + private static Map findEntriesExceedingCompressionRatio( + Path archivePath, SkillPackageLimits limits) throws IOException { + Map entries = new LinkedHashMap<>(); + try (ZipFile zipFile = new ZipFile(archivePath)) { + Enumeration archiveEntries = zipFile.getEntriesInPhysicalOrder(); + while (archiveEntries.hasMoreElements()) { + ZipArchiveEntry entry = archiveEntries.nextElement(); + long size = entry.getSize(); + long compressedSize = entry.getCompressedSize(); + if (size > 0 && (compressedSize == 0 + || ((double) size / compressedSize) > limits.getMaxCompressionRatio())) { + entries.put(entry.getName(), new StoredEntryMetadata(size, entry.getCrc())); + } + } + } + return entries; + } + + /** + * 将单个文本严格编码到当前 ZIP entry,不在输出计划中保留 UTF-8 byte 数组。 + * + * @param output 当前 ZIP 输出流 + * @param text 文本内容 + * @param path 包内文件路径 + * @throws IOException 写出失败 + */ + private void writeTextResource(OutputStream output, String text, String path) throws IOException { + output.write(strictUtf8Bytes(text, path)); + } + + /** + * 计算单个文本文件的 UTF-8 字节数;结果只用于边界校验,不在输出计划中保留 byte 数组。 + * + * @param text 文本内容 + * @param path 包内文件路径 + * @return UTF-8 字节数 + */ + private long utf8Size(String text, String path) { + try { + return SkillUtf8.byteLength(text); + } catch (CharacterCodingException e) { + throw new SkillPackageException("INVALID_UTF8", path, + "Text Skill resource must be losslessly encodable as UTF-8.", e); + } + } + + private void writeBinaryResource(OutputStream output, SkillResource resource, + SkillPackageLimits limits, String path) throws IOException { + long maxBytes = Math.min(limits.getMaxBinaryFileBytes(), resource.getSize()); + MessageDigest digest = SkillHashes.newSha256Digest(); + long count; + try (InputStream input = contentStore.open(resource.getContentRef())) { + count = transfer(input, output, maxBytes, digest); + } catch (SkillPackageException | IOException e) { + throw e; + } catch (RuntimeException e) { + throw new SkillPackageException("CONTENT_STORE_ERROR", path, + "Failed to open binary Skill resource content.", e); + } + if (count != resource.getSize()) { + throw packageError("RESOURCE_SIZE_MISMATCH", path, + "Binary resource size does not match stored content."); + } + String actualHash = SkillHashes.toHex(digest.digest()); + if (!actualHash.equals(resource.getContentHash())) { + throw packageError("RESOURCE_HASH_MISMATCH", path, + "Binary resource hash does not match stored content."); + } + } + + private boolean contentExists(String contentRef, String path) { + try { + return contentStore.exists(contentRef); + } catch (RuntimeException e) { + throw new SkillPackageException("CONTENT_STORE_ERROR", path, + "Failed to inspect binary Skill resource content.", e); + } + } + + private static String readStrictText(ZipFile zipFile, ArchiveFile file, long limit) throws IOException { + if (file.entry.getSize() > limit) { + throw packageError("TEXT_FILE_SIZE_LIMIT", file.fullPath, + "Text resource exceeds " + limit + " bytes."); + } + byte[] bytes; + try (InputStream input = zipFile.getInputStream(file.entry)) { + bytes = readBytes(input, limit, file.fullPath); + } + if (bytes.length != file.entry.getSize()) { + throw packageError("ENTRY_SIZE_MISMATCH", file.fullPath, + "ZIP entry size changed while reading."); + } + CRC32 crc = new CRC32(); + crc.update(bytes); + verifyCrc(file.entry, crc.getValue(), file.fullPath); + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + } catch (CharacterCodingException e) { + throw new SkillPackageException("INVALID_UTF8", file.fullPath, + "Text Skill resource must be valid UTF-8.", e); + } + } + + private static byte[] readBytes(InputStream input, long limit, String path) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream((int) Math.min(limit, 8_192)); + transfer(input, output, limit, null, path); + return output.toByteArray(); + } + + /** + * 将已识别为文本的内容严格编码为 UTF-8 字节。 + * + * @param text 文本内容 + * @param path 包内文件路径 + * @return UTF-8 字节 + * @throws SkillPackageException 文本包含非法 UTF-16 code unit + */ + private static byte[] strictUtf8Bytes(String text, String path) { + try { + return SkillUtf8.encode(text); + } catch (CharacterCodingException e) { + throw new SkillPackageException("INVALID_UTF8", path, + "Text Skill resource must be losslessly encodable as UTF-8.", e); + } + } + + private static InputCopyResult copyInput(InputStream input, Path target, long limit) throws IOException { + MessageDigest digest = SkillHashes.newSha256Digest(); + long count = 0; + try (OutputStream fileOutput = Files.newOutputStream(target, StandardOpenOption.TRUNCATE_EXISTING); + DigestOutputStream digestOutput = new DigestOutputStream(fileOutput, digest)) { + byte[] buffer = new byte[16 * 1024]; + int read; + while ((read = input.read(buffer)) >= 0) { + if (read == 0) { + continue; + } + if (count > limit - read) { + throw packageError("COMPRESSED_SIZE_LIMIT", null, + "Compressed Skill ZIP exceeds " + limit + " bytes."); + } + digestOutput.write(buffer, 0, read); + count += read; + } + } + if (count == 0) { + throw packageError("EMPTY_ZIP", null, "Skill ZIP input cannot be empty."); + } + return new InputCopyResult(SkillHashes.toHex(digest.digest()), count); + } + + private static void validateZipSignature(Path archivePath) throws IOException { + byte[] signature = new byte[4]; + try (InputStream input = Files.newInputStream(archivePath)) { + if (input.read(signature) != signature.length) { + throw packageError("INVALID_ZIP", null, "Skill package is not a valid ZIP archive."); + } + } + boolean localFile = signature[0] == 0x50 && signature[1] == 0x4B + && signature[2] == 0x03 && signature[3] == 0x04; + boolean emptyArchive = signature[0] == 0x50 && signature[1] == 0x4B + && signature[2] == 0x05 && signature[3] == 0x06; + if (!localFile && !emptyArchive) { + throw packageError("INVALID_ZIP", null, + "Skill package must start with a standard ZIP signature."); + } + } + + /** + * 在 Commons Compress 构造所有 entry 对象前,以常量内存扫描中央目录并执行条目数限制。 + * + * @param archivePath ZIP 临时文件 + * @param maxEntryCount 最大条目数 + * @throws SkillPackageException 中央目录非法或条目数超限 + */ + private static void enforceCentralDirectoryEntryLimit(Path archivePath, int maxEntryCount) { + try (SeekableByteChannel channel = Files.newByteChannel(archivePath, StandardOpenOption.READ)) { + CentralDirectoryDescriptor descriptor = readCentralDirectoryDescriptor(channel); + if (descriptor.entryCount > maxEntryCount) { + throw packageError("ENTRY_COUNT_LIMIT", null, + "Skill ZIP exceeds " + maxEntryCount + " entries."); + } + + long centralDirectoryEnd = Math.addExact( + descriptor.offset, descriptor.size); + if (descriptor.offset < 0 || descriptor.size < 0 + || centralDirectoryEnd != descriptor.endRecordOffset + || centralDirectoryEnd > channel.size()) { + throw invalidZip("ZIP central directory bounds are invalid."); + } + + long position = descriptor.offset; + long actualEntryCount = 0; + while (position < centralDirectoryEnd) { + if (centralDirectoryEnd - position < CENTRAL_DIRECTORY_HEADER_SIZE) { + throw invalidZip("ZIP central directory is truncated."); + } + ByteBuffer header = readAt(channel, position, CENTRAL_DIRECTORY_HEADER_SIZE); + if (header.getInt(0) != CENTRAL_DIRECTORY_SIGNATURE) { + throw invalidZip("ZIP central directory contains an invalid entry header."); + } + long variableSize = (long) unsignedShort(header, 28) + + unsignedShort(header, 30) + + unsignedShort(header, 32); + long nextPosition = Math.addExact(position, + Math.addExact((long) CENTRAL_DIRECTORY_HEADER_SIZE, variableSize)); + if (nextPosition > centralDirectoryEnd) { + throw invalidZip("ZIP central directory entry exceeds its declared bounds."); + } + actualEntryCount++; + if (actualEntryCount > maxEntryCount) { + throw packageError("ENTRY_COUNT_LIMIT", null, + "Skill ZIP exceeds " + maxEntryCount + " entries."); + } + position = nextPosition; + } + if (actualEntryCount != descriptor.entryCount) { + throw invalidZip("ZIP central directory entry count is inconsistent."); + } + } catch (SkillPackageException exception) { + throw exception; + } catch (IOException | ArithmeticException exception) { + throw new SkillPackageException("INVALID_ZIP", null, + "Skill package ZIP central directory is invalid.", exception); + } + } + + /** + * 从 EOCD 或 ZIP64 EOCD 读取受信前所需的中央目录边界信息。 + * + * @param channel ZIP 文件通道 + * @return 中央目录描述 + * @throws IOException 文件读取失败 + */ + private static CentralDirectoryDescriptor readCentralDirectoryDescriptor( + SeekableByteChannel channel) throws IOException { + long archiveSize = channel.size(); + if (archiveSize < END_OF_CENTRAL_DIRECTORY_MIN_SIZE) { + throw invalidZip("ZIP end-of-central-directory record is missing."); + } + int tailSize = (int) Math.min(archiveSize, END_OF_CENTRAL_DIRECTORY_MAX_SIZE); + long tailOffset = archiveSize - tailSize; + ByteBuffer tail = readAt(channel, tailOffset, tailSize); + int eocdIndex = findEndOfCentralDirectory(tail); + if (eocdIndex < 0) { + throw invalidZip("ZIP end-of-central-directory record is missing or malformed."); + } + + long eocdOffset = tailOffset + eocdIndex; + int diskNumber = unsignedShort(tail, eocdIndex + 4); + int centralDirectoryDisk = unsignedShort(tail, eocdIndex + 6); + int entriesOnDisk = unsignedShort(tail, eocdIndex + 8); + int totalEntries = unsignedShort(tail, eocdIndex + 10); + long centralDirectorySize = unsignedInt(tail, eocdIndex + 12); + long centralDirectoryOffset = unsignedInt(tail, eocdIndex + 16); + boolean zip64 = diskNumber == UINT16_MAX + || centralDirectoryDisk == UINT16_MAX + || entriesOnDisk == UINT16_MAX + || totalEntries == UINT16_MAX + || centralDirectorySize == UINT32_MAX + || centralDirectoryOffset == UINT32_MAX; + if (!zip64) { + if (diskNumber != 0 || centralDirectoryDisk != 0 || entriesOnDisk != totalEntries) { + throw packageError("MULTI_DISK_ZIP", null, + "Multi-disk ZIP packages are not supported."); + } + return new CentralDirectoryDescriptor( + centralDirectoryOffset, centralDirectorySize, totalEntries, eocdOffset); + } + return readZip64CentralDirectoryDescriptor(channel, eocdOffset, + diskNumber, centralDirectoryDisk, entriesOnDisk, totalEntries, + centralDirectorySize, centralDirectoryOffset); + } + + /** + * 读取并交叉校验 ZIP64 EOCD 与定位器。 + * + * @param channel ZIP 文件通道 + * @param eocdOffset 传统 EOCD 偏移 + * @param diskNumber 传统 EOCD 磁盘号 + * @param centralDirectoryDisk 传统 EOCD 中央目录磁盘号 + * @param entriesOnDisk 传统 EOCD 当前磁盘条目数 + * @param totalEntries 传统 EOCD 总条目数 + * @param centralDirectorySize 传统 EOCD 中央目录大小 + * @param centralDirectoryOffset 传统 EOCD 中央目录偏移 + * @return ZIP64 中央目录描述 + * @throws IOException 文件读取失败 + */ + private static CentralDirectoryDescriptor readZip64CentralDirectoryDescriptor( + SeekableByteChannel channel, + long eocdOffset, + int diskNumber, + int centralDirectoryDisk, + int entriesOnDisk, + int totalEntries, + long centralDirectorySize, + long centralDirectoryOffset) throws IOException { + long locatorOffset = eocdOffset - ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIZE; + if (locatorOffset < 0) { + throw invalidZip("ZIP64 end-of-central-directory locator is missing."); + } + ByteBuffer locator = readAt( + channel, locatorOffset, ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIZE); + if (locator.getInt(0) != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE) { + throw invalidZip("ZIP64 end-of-central-directory locator is missing."); + } + long locatorDisk = unsignedInt(locator, 4); + long zip64Offset = unsignedLong(locator, 8); + long totalDisks = unsignedInt(locator, 16); + if (locatorDisk != 0 || totalDisks != 1) { + throw packageError("MULTI_DISK_ZIP", null, + "Multi-disk ZIP packages are not supported."); + } + + ByteBuffer zip64 = readAt( + channel, zip64Offset, ZIP64_END_OF_CENTRAL_DIRECTORY_MIN_SIZE); + if (zip64.getInt(0) != ZIP64_END_OF_CENTRAL_DIRECTORY_SIGNATURE) { + throw invalidZip("ZIP64 end-of-central-directory record is missing."); + } + long zip64RecordSize = unsignedLong(zip64, 4); + if (zip64RecordSize < 44) { + throw invalidZip("ZIP64 end-of-central-directory record is too short."); + } + long zip64RecordEnd = Math.addExact(zip64Offset, Math.addExact(12L, zip64RecordSize)); + if (zip64RecordEnd != locatorOffset) { + throw invalidZip("ZIP64 end-of-central-directory bounds are invalid."); + } + + long zip64DiskNumber = unsignedInt(zip64, 16); + long zip64CentralDirectoryDisk = unsignedInt(zip64, 20); + long zip64EntriesOnDisk = unsignedLong(zip64, 24); + long zip64TotalEntries = unsignedLong(zip64, 32); + long zip64CentralDirectorySize = unsignedLong(zip64, 40); + long zip64CentralDirectoryOffset = unsignedLong(zip64, 48); + if (zip64DiskNumber != 0 || zip64CentralDirectoryDisk != 0 + || zip64EntriesOnDisk != zip64TotalEntries) { + throw packageError("MULTI_DISK_ZIP", null, + "Multi-disk ZIP packages are not supported."); + } + validateLegacyZip64Value(diskNumber, UINT16_MAX, zip64DiskNumber, + "ZIP disk number is inconsistent with ZIP64 metadata."); + validateLegacyZip64Value(centralDirectoryDisk, UINT16_MAX, + zip64CentralDirectoryDisk, + "ZIP central-directory disk is inconsistent with ZIP64 metadata."); + validateLegacyZip64Value(entriesOnDisk, UINT16_MAX, zip64EntriesOnDisk, + "ZIP entry count is inconsistent with ZIP64 metadata."); + validateLegacyZip64Value(totalEntries, UINT16_MAX, zip64TotalEntries, + "ZIP entry count is inconsistent with ZIP64 metadata."); + validateLegacyZip64Value(centralDirectorySize, UINT32_MAX, + zip64CentralDirectorySize, + "ZIP central-directory size is inconsistent with ZIP64 metadata."); + validateLegacyZip64Value(centralDirectoryOffset, UINT32_MAX, + zip64CentralDirectoryOffset, + "ZIP central-directory offset is inconsistent with ZIP64 metadata."); + return new CentralDirectoryDescriptor(zip64CentralDirectoryOffset, + zip64CentralDirectorySize, zip64TotalEntries, zip64Offset); + } + + /** + * 校验传统 EOCD 中未使用哨兵值的字段与 ZIP64 元数据一致。 + * + * @param legacyValue 传统字段值 + * @param sentinel ZIP64 哨兵值 + * @param zip64Value ZIP64 字段值 + * @param message 不一致时的错误消息 + */ + private static void validateLegacyZip64Value(long legacyValue, long sentinel, + long zip64Value, String message) { + if (legacyValue != sentinel && legacyValue != zip64Value) { + throw invalidZip(message); + } + } + + /** + * 在文件尾缓冲区中定位与注释长度一致的 EOCD。 + * + * @param tail ZIP 文件尾缓冲区 + * @return EOCD 相对偏移,未找到时返回 -1 + */ + private static int findEndOfCentralDirectory(ByteBuffer tail) { + for (int index = tail.limit() - END_OF_CENTRAL_DIRECTORY_MIN_SIZE; index >= 0; index--) { + if (tail.getInt(index) == END_OF_CENTRAL_DIRECTORY_SIGNATURE) { + int commentLength = unsignedShort(tail, index + 20); + if (index + END_OF_CENTRAL_DIRECTORY_MIN_SIZE + commentLength == tail.limit()) { + return index; + } + } + } + return -1; + } + + /** + * 从通道指定位置完整读取固定长度数据,并按小端序返回。 + * + * @param channel 文件通道 + * @param position 起始偏移 + * @param length 读取长度 + * @return 已翻转的小端缓冲区 + * @throws IOException 读取失败或文件截断 + */ + private static ByteBuffer readAt(SeekableByteChannel channel, long position, int length) + throws IOException { + if (position < 0 || length < 0 || position > channel.size() - length) { + throw new EOFException("ZIP record exceeds archive bounds."); + } + ByteBuffer buffer = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN); + channel.position(position); + while (buffer.hasRemaining()) { + int read = channel.read(buffer); + if (read < 0) { + throw new EOFException("ZIP record is truncated."); + } + } + buffer.flip(); + return buffer; + } + + /** + * 读取无符号 16 位小端整数。 + * + * @param buffer 小端缓冲区 + * @param offset 字段偏移 + * @return 无符号值 + */ + private static int unsignedShort(ByteBuffer buffer, int offset) { + return Short.toUnsignedInt(buffer.getShort(offset)); + } + + /** + * 读取无符号 32 位小端整数。 + * + * @param buffer 小端缓冲区 + * @param offset 字段偏移 + * @return 无符号值 + */ + private static long unsignedInt(ByteBuffer buffer, int offset) { + return Integer.toUnsignedLong(buffer.getInt(offset)); + } + + /** + * 读取当前实现可安全表示的无符号 64 位小端整数。 + * + * @param buffer 小端缓冲区 + * @param offset 字段偏移 + * @return 非负 long 值 + */ + private static long unsignedLong(ByteBuffer buffer, int offset) { + long value = buffer.getLong(offset); + if (value < 0) { + throw invalidZip("ZIP64 value exceeds the supported range."); + } + return value; + } + + /** + * 创建稳定的非法 ZIP 异常。 + * + * @param message 错误消息 + * @return 非法 ZIP 异常 + */ + private static SkillPackageException invalidZip(String message) { + return packageError("INVALID_ZIP", null, message); + } + + private static long transfer(InputStream input, OutputStream output, long limit, MessageDigest digest) + throws IOException { + return transfer(input, output, limit, digest, null); + } + + private static long transfer(InputStream input, OutputStream output, long limit, + MessageDigest digest, String path) throws IOException { + byte[] buffer = new byte[16 * 1024]; + long total = 0; + int read; + while ((read = input.read(buffer)) >= 0) { + if (read == 0) { + continue; + } + if (total > limit - read) { + throw packageError("SIZE_LIMIT", path, "Skill content exceeds " + limit + " bytes."); + } + output.write(buffer, 0, read); + if (digest != null) { + digest.update(buffer, 0, read); + } + total += read; + } + return total; + } + + private static String normalizeArchivePath(String rawPath, SkillPackageLimits limits) { + String normalized; + try { + normalized = SkillPaths.normalize(rawPath); + } catch (SkillValidationException e) { + throw packageError("UNSAFE_ENTRY_PATH", rawPath, e.getMessage()); + } + validateOutputPath(normalized, limits); + return normalized; + } + + /** + * 严格校验并返回 ZIP entry 的 UTF-8 文件名。 + * + * @param entry ZIP entry + * @return 唯一的 UTF-8 文件名 + * @throws SkillPackageException 原始文件名字节非法或存在歧义 + */ + private static String strictUtf8EntryName(ZipArchiveEntry entry) { + byte[] rawName = entry.getRawName(); + if (rawName == null) { + throw packageError("INVALID_UTF8_ENTRY_NAME", entry.getName(), + "ZIP entry name bytes are required."); + } + try { + String decodedName = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(rawName)) + .toString(); + if (!decodedName.equals(entry.getName())) { + throw packageError("INVALID_UTF8_ENTRY_NAME", entry.getName(), + "ZIP entry name must have one unambiguous UTF-8 representation."); + } + return decodedName; + } catch (CharacterCodingException e) { + throw new SkillPackageException("INVALID_UTF8_ENTRY_NAME", entry.getName(), + "ZIP entry name must be valid UTF-8.", e); + } + } + + private static String stripDirectorySuffix(String path) { + int end = path == null ? 0 : path.length(); + while (end > 0 && (path.charAt(end - 1) == '/' || path.charAt(end - 1) == '\\')) { + end--; + } + return path == null ? null : path.substring(0, end); + } + + private static void validateOutputPath(String path, SkillPackageLimits limits) { + try { + SkillUtf8.encode(path); + } catch (CharacterCodingException e) { + throw new SkillPackageException("INVALID_UTF8_PATH", path, + "Skill output path must be losslessly encodable as UTF-8.", e); + } + if (path.length() > limits.getMaxPathLength()) { + throw packageError("PATH_LENGTH_LIMIT", path, + "Skill path exceeds " + limits.getMaxPathLength() + " characters."); + } + if (SkillPaths.depth(path) > limits.getMaxPathDepth()) { + throw packageError("PATH_DEPTH_LIMIT", path, + "Skill path exceeds depth " + limits.getMaxPathDepth() + "."); + } + } + + /** + * 查找当前文件路径与已索引文件之间的祖先或后代冲突。 + * + * @param pathKey 当前路径冲突键 + * @param filePaths 已索引的文件路径 + * @param descendantPaths 已索引路径对应的首个后代文件 + * @return 冲突文件路径,不存在时返回 null + */ + private static String findHierarchyConflict(String pathKey, Map filePaths, + Map descendantPaths) { + String descendant = descendantPaths.get(pathKey); + if (descendant != null) { + return descendant; + } + int separator = pathKey.indexOf('/'); + while (separator >= 0) { + String ancestor = filePaths.get(pathKey.substring(0, separator)); + if (ancestor != null) { + return ancestor; + } + separator = pathKey.indexOf('/', separator + 1); + } + return null; + } + + /** + * 为文件路径的每个祖先建立后代索引。 + * + * @param pathKey 文件路径冲突键 + * @param path 原始文件路径 + * @param descendantPaths 后代索引 + */ + private static void indexDescendantPath(String pathKey, String path, + Map descendantPaths) { + int separator = pathKey.indexOf('/'); + while (separator >= 0) { + descendantPaths.putIfAbsent(pathKey.substring(0, separator), path); + separator = pathKey.indexOf('/', separator + 1); + } + } + + private static void checkCompressionRatio(String path, long size, long compressedSize, + SkillPackageLimits limits) { + if (size == 0) { + return; + } + if (compressedSize == 0 || ((double) size / compressedSize) > limits.getMaxCompressionRatio()) { + throw packageError("COMPRESSION_RATIO_LIMIT", path, + "ZIP entry compression ratio exceeds " + limits.getMaxCompressionRatio() + ":1."); + } + } + + private static void verifyCrc(ZipArchiveEntry entry, long actualCrc, String path) { + if (entry.getCrc() != actualCrc) { + throw packageError("CRC_MISMATCH", path, + "ZIP entry CRC does not match decompressed content."); + } + } + + private static long checkedOutputTotal(long current, long size, SkillPackageLimits limits, String path) { + if (size < 0) { + throw packageError("NEGATIVE_RESOURCE_SIZE", path, "Skill resource size cannot be negative."); + } + long total; + try { + total = Math.addExact(current, size); + } catch (ArithmeticException e) { + throw packageError("TOTAL_SIZE_LIMIT", path, "Skill package size overflow."); + } + if (total > limits.getMaxTotalUncompressedBytes()) { + throw packageError("TOTAL_SIZE_LIMIT", path, + "Skill package exceeds the total uncompressed size limit."); + } + return total; + } + + private static String detectMediaType(String path) { + String lowerPath = path.toLowerCase(Locale.ROOT); + if (lowerPath.endsWith(".md") || lowerPath.endsWith(".markdown")) { + return "text/markdown"; + } + if (lowerPath.endsWith(".yaml") || lowerPath.endsWith(".yml")) { + return "application/yaml"; + } + if (lowerPath.endsWith(".json")) { + return "application/json"; + } + if (lowerPath.endsWith(".py")) { + return "text/x-python"; + } + if (lowerPath.endsWith(".js") || lowerPath.endsWith(".mjs") || lowerPath.endsWith(".cjs")) { + return "text/javascript"; + } + if (lowerPath.endsWith(".sh") || lowerPath.endsWith(".bash") || lowerPath.endsWith(".zsh")) { + return "text/x-shellscript"; + } + String detected = URLConnection.guessContentTypeFromName(path); + return detected == null ? DEFAULT_MEDIA_TYPE : detected; + } + + private static String scalar(Object value) { + return value instanceof String text ? text : null; + } + + private static SkillPackageException validationPackageError(SkillValidationException exception) { + return validationPackageError(exception, null); + } + + /** + * 将文档校验异常转换为包异常,并按需补齐包装 Skill 根目录。 + * + * @param exception 文档校验异常 + * @param packageRoot 包装 Skill 根目录,根直压时为空 + * @return 带完整结构化报告的包异常 + */ + private static SkillPackageException validationPackageError( + SkillValidationException exception, String packageRoot) { + SkillValidationReport report = exception.getReport(); + if (report == null) { + report = new SkillValidationReport().add(new SkillValidationIssue( + exception.getCode(), SkillValidationSeverity.ERROR, exception.getPath(), + exception.getLine(), exception.getColumn(), exception.getMessage(), null)); + } + return new SkillPackageException("Invalid Skill document.", + prefixValidationPaths(report, packageRoot)); + } + + /** + * 为包装布局中的校验问题补齐 Skill 根目录,使批量预览可唯一定位文件。 + * + * @param report 原始 Skill 相对路径报告 + * @param packageRoot 包装 Skill 根目录,根直压时为空 + * @return 路径补齐后的报告 + */ + private static SkillValidationReport prefixValidationPaths( + SkillValidationReport report, String packageRoot) { + if (report == null || packageRoot == null || packageRoot.isBlank()) { + return report; + } + SkillValidationReport prefixed = new SkillValidationReport(); + for (SkillValidationIssue issue : report.getIssues()) { + String path = issue.getPath(); + if (path != null && !path.isBlank() + && !path.equals(packageRoot) && !path.startsWith(packageRoot + "/")) { + path = packageRoot + "/" + path; + } + prefixed.add(new SkillValidationIssue(issue.getCode(), issue.getSeverity(), path, + issue.getLine(), issue.getColumn(), issue.getMessage(), issue.getSuggestion())); + } + return prefixed; + } + + private static SkillPackageException packageError(String code, String path, String message) { + return new SkillPackageException(code, path, message); + } + + private static boolean hasCause(Throwable throwable, Class type) { + Throwable current = throwable; + while (current != null) { + if (type.isInstance(current)) { + return true; + } + current = current.getCause(); + } + return false; + } + + private static void deleteQuietly(Path path) { + if (path == null) { + return; + } + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + // 临时文件由操作系统兜底清理;主异常优先返回给调用方。 + } + } + + private record ArchiveFile(ZipArchiveEntry entry, String fullPath, String relativePath) { + + private ArchiveFile(ZipArchiveEntry entry, String fullPath) { + this(entry, fullPath, fullPath); + } + + private ArchiveFile withRelativePath(String value) { + return new ArchiveFile(entry, fullPath, value); + } + } + + private record ArchiveGroup(String root, List files) { + } + + private record ArchiveLayout(SkillPackageLayout layout, List groups) { + } + + private record DecodeState(SkillPackageLayout layout, List skills) { + } + + /** + * 预扫描得到的中央目录边界与条目数。 + * + * @param offset 中央目录偏移 + * @param size 中央目录大小 + * @param entryCount 声明条目数 + * @param endRecordOffset 紧随中央目录的 EOCD 或 ZIP64 EOCD 偏移 + */ + private record CentralDirectoryDescriptor(long offset, long size, long entryCount, + long endRecordOffset) { + } + + private record InputCopyResult(String hash, long size) { + } + + private record OutputWriteResult(String hash, long size) { + } + + /** + * STORED entry 写出所需的中央目录元数据。 + * + * @param size 未压缩字节数 + * @param crc CRC-32 + */ + private record StoredEntryMetadata(long size, long crc) { + } + + private static final class OutputFile { + + private final String path; + private final String text; + private final SkillResource resource; + + private OutputFile(String path, String text, SkillResource resource) { + this.path = path; + this.text = text; + this.resource = resource; + } + + private static OutputFile text(String path, String text) { + return new OutputFile(path, text, null); + } + + private static OutputFile binary(String path, SkillResource resource) { + return new OutputFile(path, null, resource); + } + } + + private static final class StageTracker { + + private final SkillContentStore contentStore; + private final List records = new ArrayList<>(); + private boolean finalized; + + private StageTracker(SkillContentStore contentStore) { + this.contentStore = contentStore; + } + + private void add(SkillContentStage stage, SkillResource resource, String path) { + records.add(new StageRecord(stage, resource, path)); + } + + private void commitAll() { + for (StageRecord record : records) { + String contentRef = contentStore.commit(record.stage); + record.committedRef = contentRef; + if (!record.stage.getContentRef().equals(contentRef)) { + throw packageError("CONTENT_REF_MISMATCH", record.path, + "Content store returned a different reference during commit."); + } + record.resource.setContentRef(contentRef); + } + finalized = true; + } + + private void rollbackAll() { + if (finalized) { + return; + } + SkillPackageException rollbackFailure = null; + for (int index = records.size() - 1; index >= 0; index--) { + try { + contentStore.rollback(records.get(index).stage); + } catch (RuntimeException cleanupError) { + if (rollbackFailure == null) { + rollbackFailure = new SkillPackageException( + "SKILL_CONTENT_ROLLBACK_ERROR", records.get(index).path, + "Failed to rollback staged Skill package content.", cleanupError); + } else { + rollbackFailure.addSuppressed(cleanupError); + } + } + } + finalized = true; + if (rollbackFailure != null) { + throw rollbackFailure; + } + } + + private void cleanupAfterFailure(Throwable primary) { + if (finalized) { + return; + } + for (int index = records.size() - 1; index >= 0; index--) { + StageRecord record = records.get(index); + try { + if (record.committedRef == null) { + contentStore.rollback(record.stage); + } else { + contentStore.release(record.committedRef); + } + } catch (RuntimeException cleanupError) { + primary.addSuppressed(cleanupError); + } + } + finalized = true; + } + } + + private static final class StageRecord { + + private final SkillContentStage stage; + private final SkillResource resource; + private final String path; + private String committedRef; + + private StageRecord(SkillContentStage stage, SkillResource resource, String path) { + this.stage = stage; + this.resource = resource; + this.path = path; + } + } + + private static final class LimitedOutputStream extends OutputStream { + + private final OutputStream delegate; + private final long limit; + private long count; + + private LimitedOutputStream(OutputStream delegate, long limit) { + this.delegate = delegate; + this.limit = limit; + } + + @Override + public void write(int value) throws IOException { + ensureCapacity(1); + delegate.write(value); + count++; + } + + @Override + public void write(byte[] bytes, int offset, int length) throws IOException { + ensureCapacity(length); + delegate.write(bytes, offset, length); + count += length; + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private long getCount() { + return count; + } + + private void ensureCapacity(int length) throws IOException { + if (count > limit - length) { + throw new CompressedSizeLimitException( + "Compressed Skill ZIP exceeds " + limit + " bytes."); + } + } + } + + private static final class CompressedSizeLimitException extends IOException { + + private CompressedSizeLimitException(String message) { + super(message); } } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java index ef388ae..0d3b6c1 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillPackageException.java @@ -1,17 +1,23 @@ package com.easyagents.skill.exception; +import com.easyagents.skill.validation.SkillValidationReport; + /** * Skill 包导入导出异常。 */ public class SkillPackageException extends SkillException { + private final String code; + private final String path; + private final SkillValidationReport report; + /** * 创建 Skill 包异常。 * * @param message 异常信息 */ public SkillPackageException(String message) { - super(message); + this("SKILL_PACKAGE_FAILED", null, message, null, null); } /** @@ -21,6 +27,62 @@ public class SkillPackageException extends SkillException { * @param cause 原始异常 */ public SkillPackageException(String message, Throwable cause) { + this("SKILL_PACKAGE_FAILED", null, message, cause, null); + } + + /** + * 创建可定位的 Skill 包异常。 + * + * @param code 稳定错误码 + * @param path 包内路径 + * @param message 异常信息 + */ + public SkillPackageException(String code, String path, String message) { + this(code, path, message, null, null); + } + + /** + * 创建包含根因的可定位 Skill 包异常。 + * + * @param code 稳定错误码 + * @param path 包内路径 + * @param message 异常信息 + * @param cause 原始异常 + */ + public SkillPackageException(String code, String path, String message, Throwable cause) { + this(code, path, message, cause, null); + } + + /** + * 创建包含结构化校验报告的 Skill 包异常。 + * + * @param message 异常信息 + * @param report 结构化校验报告 + */ + public SkillPackageException(String message, SkillValidationReport report) { + this("SKILL_PACKAGE_INVALID", null, message, null, report); + } + + private SkillPackageException(String code, String path, String message, + Throwable cause, SkillValidationReport report) { super(message, cause); + this.code = code; + this.path = path; + this.report = report; + } + + /** @return 稳定错误码 */ + public String getCode() { + return code; + } + + /** @return 包内路径 */ + public String getPath() { + return path; + } + + /** @return 结构化校验报告 */ + public SkillValidationReport getReport() { + return report; } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java index dbf781c..0612859 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/exception/SkillValidationException.java @@ -1,17 +1,25 @@ package com.easyagents.skill.exception; +import com.easyagents.skill.validation.SkillValidationReport; + /** * Skill 校验失败异常。 */ public class SkillValidationException extends SkillException { + private final String code; + private final String path; + private final Integer line; + private final Integer column; + private final SkillValidationReport report; + /** * 创建 Skill 校验异常。 * * @param message 异常信息 */ public SkillValidationException(String message) { - super(message); + this("SKILL_VALIDATION_FAILED", null, null, null, message, null, null); } /** @@ -21,6 +29,77 @@ public class SkillValidationException extends SkillException { * @param cause 原始异常 */ public SkillValidationException(String message, Throwable cause) { + this("SKILL_VALIDATION_FAILED", null, null, null, message, cause, null); + } + + /** + * 创建包含结构化报告的 Skill 校验异常。 + * + * @param message 异常信息 + * @param report 结构化校验报告 + */ + public SkillValidationException(String message, SkillValidationReport report) { + this("SKILL_VALIDATION_FAILED", null, null, null, message, null, report); + } + + /** + * 创建可定位的 Skill 校验异常。 + * + * @param code 稳定错误码 + * @param path 文件路径 + * @param line 一基行号 + * @param column 一基列号 + * @param message 异常信息 + * @param cause 原始异常 + */ + public SkillValidationException(String code, String path, Integer line, Integer column, + String message, Throwable cause) { + this(code, path, line, column, message, cause, null); + } + + /** + * 创建同时携带定位、根因和完整报告的 Skill 校验异常。 + * + * @param code 稳定错误码 + * @param path 文件路径 + * @param line 一基行号 + * @param column 一基列号 + * @param message 异常信息 + * @param cause 原始异常 + * @param report 完整结构化校验报告 + */ + public SkillValidationException(String code, String path, Integer line, Integer column, + String message, Throwable cause, SkillValidationReport report) { super(message, cause); + this.code = code; + this.path = path; + this.line = line; + this.column = column; + this.report = report; + } + + /** @return 稳定错误码 */ + public String getCode() { + return code; + } + + /** @return 文件路径 */ + public String getPath() { + return path; + } + + /** @return 一基行号 */ + public Integer getLine() { + return line; + } + + /** @return 一基列号 */ + public Integer getColumn() { + return column; + } + + /** @return 结构化校验报告 */ + public SkillValidationReport getReport() { + return report; } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java b/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java index 2a899cd..207816e 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java @@ -2,6 +2,8 @@ package com.easyagents.skill.factory; import com.easyagents.skill.model.*; import com.easyagents.skill.util.SkillFrontmatter; +import com.easyagents.skill.util.SkillResources; +import com.easyagents.skill.validation.defaults.DefaultSkillValidator; import java.util.List; import java.util.Map; @@ -25,6 +27,49 @@ public final class SkillFactory { return create(id, skillContent, null, null, null); } + /** + * 基于 SKILL.md 内容创建并严格校验正式标准 Skill。 + * + * @param id Skill ID + * @param skillContent SKILL.md 原始内容 + * @return 通过正式标准校验的 Skill 聚合 + */ + public static Skill createStrict(String id, String skillContent) { + Skill skill = create(id, skillContent); + new DefaultSkillValidator().validate(skill); + return skill; + } + + /** + * 基于 SKILL.md 内容和通用资源创建 Skill。 + * + * @param id 仓储 ID,可为空 + * @param skillContent SKILL.md 原始内容 + * @param resources 通用资源列表 + * @return Skill 聚合 + */ + public static Skill createWithResources(String id, String skillContent, List resources) { + Skill skill = create(id, skillContent); + skill.setResources(resources); + SkillResources.refreshLegacyViews(skill); + return skill; + } + + /** + * 基于 SKILL.md 和通用资源创建并严格校验正式标准 Skill。 + * + * @param id 仓储 ID,可为空 + * @param skillContent SKILL.md 原始内容 + * @param resources 通用资源列表 + * @return 通过正式标准校验的 Skill 聚合 + */ + public static Skill createWithResourcesStrict(String id, String skillContent, + List resources) { + Skill skill = createWithResources(id, skillContent, resources); + new DefaultSkillValidator().validate(skill); + return skill; + } + /** * 基于 SKILL.md 内容和资源列表创建 Skill。 * @@ -37,16 +82,29 @@ public final class SkillFactory { */ public static Skill create(String id, String skillContent, List references, List scripts, List assets) { - Map values = SkillFrontmatter.parse(skillContent); + SkillDocument document = SkillFrontmatter.parseDocument(skillContent); + Map values = document.getFrontmatter().getValues(); + String name = requiredScalar(values, "name"); + String description = requiredScalar(values, "description"); Skill skill = new Skill(); skill.setId(id); - skill.setName(values.get("name").toString()); - skill.setDescription(values.get("description").toString()); + skill.setName(name); + skill.setDescription(description); skill.setMetadata(new SkillMetadata(values)); - skill.setSkillContent(skillContent); + skill.setDocument(document); skill.setReferences(references); skill.setScripts(scripts); skill.setAssets(assets); + skill.setResources(SkillResources.canonicalResources(skill)); return skill; } + + private static String requiredScalar(Map values, String key) { + Object value = values.get(key); + if (!(value instanceof String text) || text.isBlank()) { + throw new com.easyagents.skill.exception.SkillValidationException( + "SKILL.md frontmatter " + key + " must be a non-blank string."); + } + return text; + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java index e845538..0cd7f9a 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java @@ -1,5 +1,7 @@ package com.easyagents.skill.model; +import com.easyagents.skill.util.SkillResources; + import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -12,10 +14,14 @@ public class Skill implements Serializable { private static final long serialVersionUID = 1L; private String id; + private String packageRoot; private String name; private String description; private SkillMetadata metadata = new SkillMetadata(); private String skillContent; + private SkillDocument document; + private List resources = new ArrayList<>(); + private boolean resourcesInitialized; private List references = new ArrayList<>(); private List scripts = new ArrayList<>(); private List assets = new ArrayList<>(); @@ -38,6 +44,24 @@ public class Skill implements Serializable { this.id = id; } + /** + * 获取导入包中的顶层目录名;该值不是仓储 ID。 + * + * @return 包目录名 + */ + public String getPackageRoot() { + return packageRoot; + } + + /** + * 设置导入包中的顶层目录名。 + * + * @param packageRoot 包目录名 + */ + public void setPackageRoot(String packageRoot) { + this.packageRoot = packageRoot; + } + /** * 获取名称。 * @@ -98,7 +122,7 @@ public class Skill implements Serializable { * @return SKILL.md 原始内容 */ public String getSkillContent() { - return skillContent; + return document == null ? skillContent : document.render(); } /** @@ -108,6 +132,69 @@ public class Skill implements Serializable { */ public void setSkillContent(String skillContent) { this.skillContent = skillContent; + this.document = null; + } + + /** + * 获取解析后的 SKILL.md 文档。 + * + * @return 解析后的文档 + */ + public SkillDocument getDocument() { + return document; + } + + /** + * 设置解析后的 SKILL.md 文档。 + * + * @param document 解析后的文档 + */ + public void setDocument(SkillDocument document) { + this.document = document; + this.skillContent = document == null ? null : document.render(); + if (document != null) { + java.util.Map values = document.getFrontmatter().getValues(); + this.metadata = new SkillMetadata(values); + this.name = values.get("name") instanceof String text ? text : null; + this.description = values.get("description") instanceof String text ? text : null; + } + } + + /** + * 获取通用资源列表。 + * + * @return 通用资源列表 + */ + public List getResources() { + if (!resourcesInitialized) { + resources = resources == null || resources.isEmpty() + ? new ArrayList<>(SkillResources.fromLegacyViews(this)) + : new ArrayList<>(resources); + resourcesInitialized = true; + } + return resources; + } + + /** + * 设置通用资源列表。 + * + * @param resources 通用资源列表 + */ + public void setResources(List resources) { + this.resources = resources == null ? new ArrayList<>() : new ArrayList<>(resources); + this.resourcesInitialized = true; + } + + /** + * 判断正式通用资源列表是否已被显式初始化。 + * + *

该标记用于区分“尚未迁移的旧资源视图”和“调用方明确设置的空资源列表”, + * 避免删除最后一个正式资源后又从旧兼容视图恢复该资源。

+ * + * @return 已显式设置通用资源列表时为 true + */ + public boolean isResourcesInitialized() { + return resourcesInitialized; } /** diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java index e5bfda1..796d7c7 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java @@ -3,8 +3,11 @@ package com.easyagents.skill.model; import java.io.Serializable; /** - * Skill 静态资产。 + * Skill 静态资产兼容视图。 + * + * @deprecated 请使用 {@link SkillResource}。 */ +@Deprecated public class SkillAsset implements Serializable { private static final long serialVersionUID = 1L; diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDocument.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDocument.java new file mode 100644 index 0000000..9ac34ac --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDocument.java @@ -0,0 +1,207 @@ +package com.easyagents.skill.model; + +import com.easyagents.skill.util.SkillFrontmatter; +import com.easyagents.skill.validation.SkillValidationIssue; + +import java.io.Serial; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 解析后的 SKILL.md 文档,保留原文并支持确定性重建。 + */ +public class SkillDocument implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private String rawContent; + private SkillMetadata frontmatter = new SkillMetadata(); + private String markdownBody = ""; + private Map frontmatterLocations = new LinkedHashMap<>(); + private List diagnostics = new ArrayList<>(); + private boolean modified; + + /** + * 创建空文档。 + */ + public SkillDocument() { + } + + /** + * 创建已解析文档。 + * + * @param rawContent 原始完整内容 + * @param frontmatter frontmatter 有序对象 + * @param markdownBody Markdown 正文 + */ + public SkillDocument(String rawContent, Map frontmatter, String markdownBody) { + this(rawContent, frontmatter, markdownBody, Map.of(), List.of()); + } + + /** + * 创建包含源码定位和解析诊断的已解析文档。 + * + * @param rawContent 原始完整内容 + * @param frontmatter frontmatter 有序对象 + * @param markdownBody Markdown 正文 + * @param frontmatterLocations 顶层 frontmatter 字段源码位置 + * @param diagnostics 解析阶段产生的结构化诊断 + */ + public SkillDocument(String rawContent, Map frontmatter, String markdownBody, + Map frontmatterLocations, + List diagnostics) { + this.rawContent = rawContent; + this.frontmatter = new SkillMetadata(frontmatter); + this.markdownBody = markdownBody == null ? "" : markdownBody; + this.frontmatterLocations = frontmatterLocations == null + ? new LinkedHashMap<>() : new LinkedHashMap<>(frontmatterLocations); + this.diagnostics = diagnostics == null ? new ArrayList<>() : new ArrayList<>(diagnostics); + } + + /** + * 获取原始完整内容。 + * + * @return 原始内容 + */ + public String getRawContent() { + return rawContent; + } + + /** + * 设置原始完整内容。 + * + * @param rawContent 原始内容 + */ + public void setRawContent(String rawContent) { + this.rawContent = rawContent; + } + + /** + * 获取 frontmatter 防御性副本;修改文档请使用 {@link #putFrontmatter(String, Object)}、 + * {@link #removeFrontmatter(String)} 或 {@link #setFrontmatter(SkillMetadata)}。 + * + * @return frontmatter 有序对象副本 + */ + public SkillMetadata getFrontmatter() { + return new SkillMetadata(frontmatter.getValues()); + } + + /** + * 设置 frontmatter 并标记文档已修改。 + * + * @param frontmatter frontmatter 有序对象 + */ + public void setFrontmatter(SkillMetadata frontmatter) { + this.frontmatter = frontmatter == null + ? new SkillMetadata() : new SkillMetadata(frontmatter.getValues()); + this.frontmatterLocations.clear(); + this.modified = true; + } + + /** + * 写入一个 frontmatter 字段并标记文档已修改。 + * + * @param key 字段名 + * @param value 字段值 + */ + public void putFrontmatter(String key, Object value) { + frontmatter.put(key, value); + frontmatterLocations.remove(key); + modified = true; + } + + /** + * 删除一个 frontmatter 字段并标记文档已修改。 + * + * @param key 字段名 + * @return 被删除值的防御性副本 + */ + public Object removeFrontmatter(String key) { + Object removed = frontmatter.remove(key); + frontmatterLocations.remove(key); + modified = true; + return removed; + } + + /** + * 获取顶层 frontmatter 字段的源码位置。 + * + * @param key frontmatter 字段名 + * @return 源码位置;字段不存在或已被结构化修改时为空 + */ + public SkillSourceLocation getFrontmatterLocation(String key) { + return key == null ? null : frontmatterLocations.get(key); + } + + /** + * 获取顶层 frontmatter 字段源码位置的不可变副本。 + * + * @return 字段与源码位置映射 + */ + public Map getFrontmatterLocations() { + return Collections.unmodifiableMap(new LinkedHashMap<>(frontmatterLocations)); + } + + /** + * 获取解析阶段结构化诊断的不可变副本。 + * + * @return 解析诊断 + */ + public List getDiagnostics() { + return Collections.unmodifiableList(new ArrayList<>(diagnostics)); + } + + /** + * 获取 Markdown 正文。 + * + * @return Markdown 正文 + */ + public String getMarkdownBody() { + return markdownBody; + } + + /** + * 设置 Markdown 正文并标记文档已修改。 + * + * @param markdownBody Markdown 正文 + */ + public void setMarkdownBody(String markdownBody) { + this.markdownBody = markdownBody == null ? "" : markdownBody; + this.modified = true; + } + + /** + * 判断结构化内容是否被修改。 + * + * @return 已修改时为 true + */ + public boolean isModified() { + return modified; + } + + /** + * 设置修改标记。 + * + * @param modified 修改标记 + */ + public void setModified(boolean modified) { + this.modified = modified; + } + + /** + * 渲染完整 SKILL.md;未修改时优先返回原文。 + * + * @return 完整 SKILL.md + */ + public String render() { + if (!modified && rawContent != null) { + return rawContent; + } + return SkillFrontmatter.serialize(frontmatter.getValues(), markdownBody); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java index f05e768..722a703 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillMetadata.java @@ -1,7 +1,14 @@ package com.easyagents.skill.model; +import com.easyagents.skill.exception.SkillValidationException; + import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Date; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -29,12 +36,12 @@ public class SkillMetadata implements Serializable { } /** - * 获取元数据键值。 + * 获取元数据键值的递归防御性副本。 * - * @return 元数据键值 + * @return 元数据键值副本 */ public Map getValues() { - return values; + return deepCopyMap(values); } /** @@ -43,7 +50,7 @@ public class SkillMetadata implements Serializable { * @param values 元数据键值 */ public void setValues(Map values) { - this.values = values == null ? new LinkedHashMap<>() : new LinkedHashMap<>(values); + this.values = values == null ? new LinkedHashMap<>() : deepCopyMap(values); } /** @@ -53,9 +60,10 @@ public class SkillMetadata implements Serializable { * @param value 元数据值 */ public void put(String key, Object value) { - if (key != null && !key.isBlank()) { - values.put(key, value); + if (key == null || key.isBlank()) { + throw new SkillValidationException("Skill metadata key must be a non-blank string."); } + values.put(key, deepCopyValue(value)); } /** @@ -65,7 +73,17 @@ public class SkillMetadata implements Serializable { * @return 元数据值 */ public Object get(String key) { - return values.get(key); + return deepCopyValue(values.get(key)); + } + + /** + * 删除一个元数据键。 + * + * @param key 元数据键 + * @return 被删除值的防御性副本 + */ + public Object remove(String key) { + return deepCopyValue(values.remove(key)); } /** @@ -76,4 +94,37 @@ public class SkillMetadata implements Serializable { public boolean isEmpty() { return values.isEmpty(); } + + private static LinkedHashMap deepCopyMap(Map source) { + LinkedHashMap target = new LinkedHashMap<>(); + for (Map.Entry entry : source.entrySet()) { + if (!(entry.getKey() instanceof String key) || key.isBlank()) { + throw new SkillValidationException("Skill metadata keys must be non-blank strings."); + } + target.put(key, deepCopyValue(entry.getValue())); + } + return target; + } + + private static Object deepCopyValue(Object value) { + if (value == null || value instanceof String || value instanceof Boolean + || value instanceof Byte || value instanceof Short || value instanceof Integer + || value instanceof Long || value instanceof Float || value instanceof Double + || value instanceof BigInteger || value instanceof BigDecimal || value instanceof Character) { + return value; + } + if (value instanceof Date date) { + return new Date(date.getTime()); + } + if (value instanceof Map map) { + return deepCopyMap(map); + } + if (value instanceof List list) { + List copy = new ArrayList<>(list.size()); + list.forEach(item -> copy.add(deepCopyValue(item))); + return copy; + } + throw new SkillValidationException("Unsupported Skill metadata value type: " + + value.getClass().getName()); + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackage.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackage.java new file mode 100644 index 0000000..179d007 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackage.java @@ -0,0 +1,71 @@ +package com.easyagents.skill.model; + +import java.io.Serial; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * 可移植 Skill 包聚合。 + */ +public class SkillPackage implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private SkillPackageLayout layout = SkillPackageLayout.SINGLE_DIRECTORY; + private List skills = new ArrayList<>(); + + /** + * 创建空 Skill 包。 + */ + public SkillPackage() { + } + + /** + * 创建 Skill 包。 + * + * @param layout 包结构形态 + * @param skills Skill 列表 + */ + public SkillPackage(SkillPackageLayout layout, List skills) { + setLayout(layout); + setSkills(skills); + } + + /** + * 获取包结构形态。 + * + * @return 包结构形态 + */ + public SkillPackageLayout getLayout() { + return layout; + } + + /** + * 设置包结构形态。 + * + * @param layout 包结构形态 + */ + public void setLayout(SkillPackageLayout layout) { + this.layout = layout == null ? SkillPackageLayout.SINGLE_DIRECTORY : layout; + } + + /** + * 获取 Skill 列表。 + * + * @return Skill 列表 + */ + public List getSkills() { + return skills; + } + + /** + * 设置 Skill 列表。 + * + * @param skills Skill 列表 + */ + public void setSkills(List skills) { + this.skills = skills == null ? new ArrayList<>() : new ArrayList<>(skills); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java new file mode 100644 index 0000000..07f58a1 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java @@ -0,0 +1,16 @@ +package com.easyagents.skill.model; + +/** + * Skill ZIP 的结构形态。 + */ +public enum SkillPackageLayout { + + /** 根目录直接包含 SKILL.md 的单 Skill 包。 */ + ROOT_SKILL, + + /** 一个顶层目录包装的单 Skill 包。 */ + SINGLE_DIRECTORY, + + /** 多个顶层 Skill 目录组成的批量包。 */ + MULTI_DIRECTORY +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLimits.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLimits.java new file mode 100644 index 0000000..7e4946b --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLimits.java @@ -0,0 +1,274 @@ +package com.easyagents.skill.model; + +import com.easyagents.skill.exception.SkillValidationException; + +import java.io.Serial; +import java.io.Serializable; + +/** + * Skill 包读取安全限额;调用方只能在底层硬上限内收紧或调整。 + */ +public final class SkillPackageLimits implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private static final int HARD_MAX_ENTRY_COUNT = 10_000; + private static final long HARD_MAX_TEXT_FILE_BYTES = 8L * 1024 * 1024; + private static final long HARD_MAX_BINARY_FILE_BYTES = 200L * 1024 * 1024; + private static final long HARD_MAX_TOTAL_BYTES = 512L * 1024 * 1024; + private static final long HARD_MAX_COMPRESSED_BYTES = 512L * 1024 * 1024; + private static final int HARD_MAX_PATH_LENGTH = 1_024; + private static final int HARD_MAX_PATH_DEPTH = 32; + private static final double HARD_MAX_COMPRESSION_RATIO = 200D; + private static final long HARD_MAX_FRONTMATTER_BYTES = 256L * 1024; + private static final int HARD_MAX_YAML_ALIASES = 50; + private static final int HARD_MAX_YAML_DEPTH = 20; + private static final int HARD_MAX_YAML_CODE_POINTS = 262_144; + + private final int maxEntryCount; + private final long maxTextFileBytes; + private final long maxBinaryFileBytes; + private final long maxTotalUncompressedBytes; + private final long maxCompressedPackageBytes; + private final int maxPathLength; + private final int maxPathDepth; + private final double maxCompressionRatio; + private final long maxFrontmatterBytes; + private final int maxYamlAliases; + private final int maxYamlDepth; + private final int maxYamlCodePoints; + + private SkillPackageLimits(Builder builder) { + maxEntryCount = positive(builder.maxEntryCount, HARD_MAX_ENTRY_COUNT, "maxEntryCount"); + maxTextFileBytes = positive(builder.maxTextFileBytes, HARD_MAX_TEXT_FILE_BYTES, "maxTextFileBytes"); + maxBinaryFileBytes = positive(builder.maxBinaryFileBytes, HARD_MAX_BINARY_FILE_BYTES, + "maxBinaryFileBytes"); + maxTotalUncompressedBytes = positive(builder.maxTotalUncompressedBytes, HARD_MAX_TOTAL_BYTES, + "maxTotalUncompressedBytes"); + maxCompressedPackageBytes = positive(builder.maxCompressedPackageBytes, HARD_MAX_COMPRESSED_BYTES, + "maxCompressedPackageBytes"); + maxPathLength = positive(builder.maxPathLength, HARD_MAX_PATH_LENGTH, "maxPathLength"); + maxPathDepth = positive(builder.maxPathDepth, HARD_MAX_PATH_DEPTH, "maxPathDepth"); + maxCompressionRatio = positive(builder.maxCompressionRatio, HARD_MAX_COMPRESSION_RATIO, + "maxCompressionRatio"); + maxFrontmatterBytes = positive(builder.maxFrontmatterBytes, HARD_MAX_FRONTMATTER_BYTES, + "maxFrontmatterBytes"); + maxYamlAliases = nonNegative(builder.maxYamlAliases, HARD_MAX_YAML_ALIASES, "maxYamlAliases"); + maxYamlDepth = positive(builder.maxYamlDepth, HARD_MAX_YAML_DEPTH, "maxYamlDepth"); + maxYamlCodePoints = positive(builder.maxYamlCodePoints, HARD_MAX_YAML_CODE_POINTS, + "maxYamlCodePoints"); + if (maxTextFileBytes > maxTotalUncompressedBytes || maxBinaryFileBytes > maxTotalUncompressedBytes) { + throw new SkillValidationException("Single-file limits cannot exceed the total uncompressed limit."); + } + } + + /** + * 创建默认限额。 + * + * @return 默认限额 + */ + public static SkillPackageLimits defaults() { + return builder().build(); + } + + /** + * 创建限额构建器。 + * + * @return 限额构建器 + */ + public static Builder builder() { + return new Builder(); + } + + /** @return 最大 ZIP entry 数 */ + public int getMaxEntryCount() { + return maxEntryCount; + } + + /** @return 单文本文件最大字节数 */ + public long getMaxTextFileBytes() { + return maxTextFileBytes; + } + + /** @return 单二进制文件最大字节数 */ + public long getMaxBinaryFileBytes() { + return maxBinaryFileBytes; + } + + /** @return 包总未压缩最大字节数 */ + public long getMaxTotalUncompressedBytes() { + return maxTotalUncompressedBytes; + } + + /** @return 输入 ZIP 最大压缩字节数 */ + public long getMaxCompressedPackageBytes() { + return maxCompressedPackageBytes; + } + + /** @return 单路径最大字符数 */ + public int getMaxPathLength() { + return maxPathLength; + } + + /** @return 最大目录深度 */ + public int getMaxPathDepth() { + return maxPathDepth; + } + + /** @return 单 entry 最大压缩比 */ + public double getMaxCompressionRatio() { + return maxCompressionRatio; + } + + /** @return frontmatter 最大字节数 */ + public long getMaxFrontmatterBytes() { + return maxFrontmatterBytes; + } + + /** @return YAML collection 最大 alias 数 */ + public int getMaxYamlAliases() { + return maxYamlAliases; + } + + /** @return YAML 最大嵌套深度 */ + public int getMaxYamlDepth() { + return maxYamlDepth; + } + + /** @return YAML 最大 Unicode code point 数 */ + public int getMaxYamlCodePoints() { + return maxYamlCodePoints; + } + + private static int positive(int value, int hardMax, String name) { + if (value <= 0 || value > hardMax) { + throw new SkillValidationException(name + " must be between 1 and " + hardMax + "."); + } + return value; + } + + private static int nonNegative(int value, int hardMax, String name) { + if (value < 0 || value > hardMax) { + throw new SkillValidationException(name + " must be between 0 and " + hardMax + "."); + } + return value; + } + + private static long positive(long value, long hardMax, String name) { + if (value <= 0 || value > hardMax) { + throw new SkillValidationException(name + " must be between 1 and " + hardMax + "."); + } + return value; + } + + private static double positive(double value, double hardMax, String name) { + if (!Double.isFinite(value) || value <= 0D || value > hardMax) { + throw new SkillValidationException(name + " must be between 0 and " + hardMax + "."); + } + return value; + } + + /** + * Skill 包限额构建器。 + */ + public static final class Builder { + + private int maxEntryCount = 2_000; + private long maxTextFileBytes = 2L * 1024 * 1024; + private long maxBinaryFileBytes = 50L * 1024 * 1024; + private long maxTotalUncompressedBytes = 200L * 1024 * 1024; + private long maxCompressedPackageBytes = 100L * 1024 * 1024; + private int maxPathLength = 512; + private int maxPathDepth = 16; + private double maxCompressionRatio = 100D; + private long maxFrontmatterBytes = 64L * 1024; + private int maxYamlAliases = 10; + private int maxYamlDepth = 10; + private int maxYamlCodePoints = 65_536; + + private Builder() { + } + + /** @param value 最大 entry 数 @return 当前构建器 */ + public Builder maxEntryCount(int value) { + maxEntryCount = value; + return this; + } + + /** @param value 单文本文件最大字节数 @return 当前构建器 */ + public Builder maxTextFileBytes(long value) { + maxTextFileBytes = value; + return this; + } + + /** @param value 单二进制文件最大字节数 @return 当前构建器 */ + public Builder maxBinaryFileBytes(long value) { + maxBinaryFileBytes = value; + return this; + } + + /** @param value 包总未压缩最大字节数 @return 当前构建器 */ + public Builder maxTotalUncompressedBytes(long value) { + maxTotalUncompressedBytes = value; + return this; + } + + /** @param value 输入 ZIP 最大压缩字节数 @return 当前构建器 */ + public Builder maxCompressedPackageBytes(long value) { + maxCompressedPackageBytes = value; + return this; + } + + /** @param value 单路径最大字符数 @return 当前构建器 */ + public Builder maxPathLength(int value) { + maxPathLength = value; + return this; + } + + /** @param value 最大目录深度 @return 当前构建器 */ + public Builder maxPathDepth(int value) { + maxPathDepth = value; + return this; + } + + /** @param value 单 entry 最大压缩比 @return 当前构建器 */ + public Builder maxCompressionRatio(double value) { + maxCompressionRatio = value; + return this; + } + + /** @param value frontmatter 最大字节数 @return 当前构建器 */ + public Builder maxFrontmatterBytes(long value) { + maxFrontmatterBytes = value; + return this; + } + + /** @param value YAML alias 最大数量 @return 当前构建器 */ + public Builder maxYamlAliases(int value) { + maxYamlAliases = value; + return this; + } + + /** @param value YAML 最大嵌套深度 @return 当前构建器 */ + public Builder maxYamlDepth(int value) { + maxYamlDepth = value; + return this; + } + + /** @param value YAML 最大 code point 数 @return 当前构建器 */ + public Builder maxYamlCodePoints(int value) { + maxYamlCodePoints = value; + return this; + } + + /** + * 构建并校验限额。 + * + * @return 不可变限额 + */ + public SkillPackageLimits build() { + return new SkillPackageLimits(this); + } + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java index c27af7e..80657d1 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java @@ -3,8 +3,11 @@ package com.easyagents.skill.model; import java.io.Serializable; /** - * Skill Markdown 参考文档。 + * Skill Markdown 参考文档兼容视图。 + * + * @deprecated 请使用 {@link SkillResource}。 */ +@Deprecated public class SkillReference implements Serializable { private static final long serialVersionUID = 1L; diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java new file mode 100644 index 0000000..cc950fe --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java @@ -0,0 +1,175 @@ +package com.easyagents.skill.model; + +import java.io.Serial; +import java.io.Serializable; + +/** + * Skill 根目录下的通用安全资源。 + */ +public class SkillResource implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private String path; + private SkillResourceKind kind = SkillResourceKind.OTHER; + private String mediaType; + private String textContent; + private String contentRef; + private String contentHash; + private long size; + private SkillMetadata metadata = new SkillMetadata(); + + /** + * 获取 Skill 根目录相对路径。 + * + * @return 资源路径 + */ + public String getPath() { + return path; + } + + /** + * 设置 Skill 根目录相对路径。 + * + * @param path 资源路径 + */ + public void setPath(String path) { + this.path = path; + } + + /** + * 获取资源语义类型。 + * + * @return 资源语义类型 + */ + public SkillResourceKind getKind() { + return kind; + } + + /** + * 设置资源语义类型。 + * + * @param kind 资源语义类型 + */ + public void setKind(SkillResourceKind kind) { + this.kind = kind == null ? SkillResourceKind.OTHER : kind; + } + + /** + * 获取媒体类型。 + * + * @return 媒体类型 + */ + public String getMediaType() { + return mediaType; + } + + /** + * 设置媒体类型。 + * + * @param mediaType 媒体类型 + */ + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + /** + * 获取严格 UTF-8 文本内容。 + * + * @return 文本内容,二进制资源返回 null + */ + public String getTextContent() { + return textContent; + } + + /** + * 设置严格 UTF-8 文本内容。 + * + * @param textContent 文本内容 + */ + public void setTextContent(String textContent) { + this.textContent = textContent; + } + + /** + * 获取二进制内容引用。 + * + * @return 内容引用,文本资源通常返回 null + */ + public String getContentRef() { + return contentRef; + } + + /** + * 设置二进制内容引用。 + * + * @param contentRef 内容引用 + */ + public void setContentRef(String contentRef) { + this.contentRef = contentRef; + } + + /** + * 获取内容 SHA-256。 + * + * @return 十六进制 SHA-256 + */ + public String getContentHash() { + return contentHash; + } + + /** + * 设置内容 SHA-256。 + * + * @param contentHash 十六进制 SHA-256 + */ + public void setContentHash(String contentHash) { + this.contentHash = contentHash; + } + + /** + * 获取原始字节大小。 + * + * @return 字节大小 + */ + public long getSize() { + return size; + } + + /** + * 设置原始字节大小。 + * + * @param size 字节大小 + */ + public void setSize(long size) { + this.size = size; + } + + /** + * 获取资源扩展元数据。 + * + * @return 扩展元数据 + */ + public SkillMetadata getMetadata() { + return metadata; + } + + /** + * 设置资源扩展元数据。 + * + * @param metadata 扩展元数据 + */ + public void setMetadata(SkillMetadata metadata) { + this.metadata = metadata == null ? new SkillMetadata() : metadata; + } + + /** + * 判断资源是否以内联文本保存。 + * + * @return 包含文本内容时为 true + */ + public boolean isText() { + return textContent != null; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResourceKind.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResourceKind.java new file mode 100644 index 0000000..59ac451 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResourceKind.java @@ -0,0 +1,22 @@ +package com.easyagents.skill.model; + +/** + * Skill 资源语义类型。 + */ +public enum SkillResourceKind { + + /** 参考资料。 */ + REFERENCE, + + /** 脚本源码,仅保存且不执行。 */ + SCRIPT, + + /** 静态资产。 */ + ASSET, + + /** 使用示例。 */ + EXAMPLE, + + /** 其他安全资源。 */ + OTHER +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java index edab344..4679906 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java @@ -3,8 +3,11 @@ package com.easyagents.skill.model; import java.io.Serializable; /** - * Skill 脚本源码。 + * Skill 脚本源码兼容视图。 + * + * @deprecated 请使用 {@link SkillResource}。 */ +@Deprecated public class SkillScript implements Serializable { private static final long serialVersionUID = 1L; diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillSourceLocation.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillSourceLocation.java new file mode 100644 index 0000000..4b0a5a2 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillSourceLocation.java @@ -0,0 +1,40 @@ +package com.easyagents.skill.model; + +import java.io.Serial; +import java.io.Serializable; + +/** + * SKILL.md 中的一基源码位置。 + */ +public final class SkillSourceLocation implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private final int line; + private final int column; + + /** + * 创建源码位置。 + * + * @param line 一基行号 + * @param column 一基列号 + */ + public SkillSourceLocation(int line, int column) { + if (line < 1 || column < 1) { + throw new IllegalArgumentException("Skill source location must be one-based."); + } + this.line = line; + this.column = column; + } + + /** @return 一基行号 */ + public int getLine() { + return line; + } + + /** @return 一基列号 */ + public int getColumn() { + return column; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java index 60795d1..867e1b4 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java @@ -92,16 +92,54 @@ public class InMemorySkillRepository implements SkillRepository { private static Skill copySkill(Skill source) { Skill target = new Skill(); target.setId(source.getId()); + target.setPackageRoot(source.getPackageRoot()); target.setName(source.getName()); target.setDescription(source.getDescription()); target.setMetadata(copyMetadata(source.getMetadata())); - target.setSkillContent(source.getSkillContent()); + SkillDocument copiedDocument = copyDocument(source.getDocument()); + if (copiedDocument == null) { + target.setSkillContent(source.getSkillContent()); + } else { + target.setDocument(copiedDocument); + } + target.setResources(copyResources(source.getResources())); target.setReferences(copyReferences(source.getReferences())); target.setScripts(copyScripts(source.getScripts())); target.setAssets(copyAssets(source.getAssets())); return target; } + private static SkillDocument copyDocument(SkillDocument source) { + if (source == null) { + return null; + } + SkillDocument target = new SkillDocument(source.getRawContent(), + source.getFrontmatter().getValues(), source.getMarkdownBody(), + source.getFrontmatterLocations(), source.getDiagnostics()); + target.setModified(source.isModified()); + return target; + } + + private static List copyResources(List sources) { + List targets = new ArrayList<>(); + if (sources == null) { + return targets; + } + for (SkillResource source : sources) { + SkillResource target = new SkillResource(); + target.setPath(source.getPath()); + target.setKind(source.getKind()); + target.setMediaType(source.getMediaType()); + target.setTextContent(source.getTextContent()); + target.setContentRef(source.getContentRef()); + target.setContentHash(source.getContentHash()); + target.setSize(source.getSize()); + target.setMetadata(copyMetadata(source.getMetadata())); + targets.add(target); + } + return targets; + } + private static List copyReferences(List sources) { List targets = new ArrayList<>(); if (sources == null) { diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java new file mode 100644 index 0000000..c94e138 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java @@ -0,0 +1,76 @@ +package com.easyagents.skill.store; + +import java.nio.file.Path; +import java.util.Objects; + +/** + * 尚未提交的 Skill 二进制内容阶段结果。 + */ +public final class SkillContentStage { + + private final String stageId; + private final String contentRef; + private final String contentHash; + private final long size; + private final boolean alreadyCommitted; + private final Path compatibilityPath; + + /** + * 创建内容阶段结果。 + * + * @param stageId 暂存标识 + * @param contentRef 最终内容引用 + * @param contentHash SHA-256 + * @param size 字节大小 + * @param alreadyCommitted 是否由兼容实现提前写入正式存储 + */ + public SkillContentStage(String stageId, String contentRef, String contentHash, + long size, boolean alreadyCommitted) { + this(stageId, contentRef, contentHash, size, alreadyCommitted, null); + } + + private SkillContentStage(String stageId, String contentRef, String contentHash, + long size, boolean alreadyCommitted, Path compatibilityPath) { + this.stageId = Objects.requireNonNull(stageId, "stageId"); + this.contentRef = Objects.requireNonNull(contentRef, "contentRef"); + this.contentHash = Objects.requireNonNull(contentHash, "contentHash"); + this.size = size; + this.alreadyCommitted = alreadyCommitted; + this.compatibilityPath = compatibilityPath; + } + + /** @return 暂存标识 */ + public String getStageId() { + return stageId; + } + + /** @return 最终内容引用 */ + public String getContentRef() { + return contentRef; + } + + /** @return SHA-256 */ + public String getContentHash() { + return contentHash; + } + + /** @return 字节大小 */ + public long getSize() { + return size; + } + + /** @return 已提前提交时为 true */ + public boolean isAlreadyCommitted() { + return alreadyCommitted; + } + + static SkillContentStage compatibility(Path path, String contentHash, long size) { + String contentRef = "sha256:" + contentHash; + return new SkillContentStage(contentRef, contentRef, contentHash, + size, false, path); + } + + Path compatibilityPath() { + return compatibilityPath; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java index e712c4e..99f39af 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java @@ -1,9 +1,19 @@ package com.easyagents.skill.store; +import com.easyagents.skill.exception.SkillException; +import com.easyagents.skill.util.SkillHashes; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.DigestOutputStream; +import java.security.MessageDigest; /** - * Skill 二进制内容存储接口。 + * Skill 二进制内容流式存储与引用生命周期接口。 */ public interface SkillContentStore { @@ -15,6 +25,123 @@ public interface SkillContentStore { */ String put(byte[] bytes); + /** + * 流式保存内容并返回内容引用。 + * + *

兼容默认实现仅缓存单个文件;正式持久化实现应覆盖该方法以直接流式写入。

+ * + * @param inputStream 内容流,不由本方法关闭 + * @param maxBytes 最大允许字节数 + * @return 内容引用 + */ + default String put(InputStream inputStream, long maxBytes) { + return put(readBounded(inputStream, maxBytes)); + } + + /** + * 暂存内容,供完成全包校验后统一提交。 + * + *

为兼容旧实现,默认实现会立即写入;正式实现应覆盖并提供真实暂存区。

+ * + * @param inputStream 内容流,不由本方法关闭 + * @param maxBytes 最大允许字节数 + * @return 暂存结果 + */ + default SkillContentStage stage(InputStream inputStream, long maxBytes) { + if (inputStream == null || maxBytes < 0) { + throw new SkillException("Valid Skill content stream and limit are required."); + } + Path temporaryFile = null; + try { + temporaryFile = Files.createTempFile("easy-agents-skill-content-", ".stage"); + MessageDigest digest = SkillHashes.newSha256Digest(); + long total = 0; + try (DigestOutputStream output = new DigestOutputStream( + Files.newOutputStream(temporaryFile, StandardOpenOption.TRUNCATE_EXISTING), digest)) { + byte[] buffer = new byte[16 * 1024]; + int read; + while ((read = inputStream.read(buffer)) >= 0) { + if (read == 0) { + continue; + } + if (total > maxBytes - read) { + throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); + } + output.write(buffer, 0, read); + total += read; + } + } + return SkillContentStage.compatibility( + temporaryFile, SkillHashes.toHex(digest.digest()), total); + } catch (IOException | RuntimeException e) { + deleteTemporaryFile(temporaryFile); + if (e instanceof SkillException skillException) { + throw skillException; + } + throw new SkillException("Failed to stage Skill content stream.", e); + } + } + + /** + * 提交暂存内容。 + * + * @param stage 暂存结果 + * @return 最终内容引用 + */ + default String commit(SkillContentStage stage) { + if (stage == null) { + throw new SkillException("Skill content stage is required."); + } + Path compatibilityPath = stage.compatibilityPath(); + if (compatibilityPath != null) { + try (InputStream input = Files.newInputStream(compatibilityPath)) { + String contentRef = put(input, stage.getSize()); + if (!stage.getContentRef().equals(contentRef)) { + release(contentRef); + throw new SkillException("Skill content store returned a non-hash content reference."); + } + return contentRef; + } catch (IOException e) { + throw new SkillException("Failed to commit staged Skill content.", e); + } finally { + deleteTemporaryFile(compatibilityPath); + } + } + return stage.getContentRef(); + } + + /** + * 回滚尚未提交的内容。 + * + * @param stage 暂存结果 + */ + default void rollback(SkillContentStage stage) { + if (stage != null) { + deleteTemporaryFile(stage.compatibilityPath()); + if (stage.isAlreadyCommitted()) { + release(stage.getContentRef()); + } + } + } + + /** + * 增加正式内容引用计数。 + * + * @param contentRef 内容引用 + */ + default void retain(String contentRef) { + // 旧实现没有引用计数,保留兼容空操作。 + } + + /** + * 释放正式内容引用;引用归零后实现可以删除物理内容。 + * + * @param contentRef 内容引用 + */ + default void release(String contentRef) { + // 旧实现没有引用计数,保留兼容空操作。 + } + /** * 打开内容流。 * @@ -38,4 +165,43 @@ public interface SkillContentStore { * @return 存在时为 true */ boolean exists(String contentRef); + + private static byte[] readBounded(InputStream inputStream, long maxBytes) { + if (inputStream == null) { + throw new SkillException("Skill content input stream is required."); + } + if (maxBytes < 0) { + throw new SkillException("Skill content max bytes cannot be negative."); + } + try { + ByteArrayOutputStream output = new ByteArrayOutputStream((int) Math.min(maxBytes, 8_192)); + byte[] buffer = new byte[8_192]; + long total = 0; + int read; + while ((read = inputStream.read(buffer)) >= 0) { + if (read == 0) { + continue; + } + total += read; + if (total > maxBytes) { + throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); + } + output.write(buffer, 0, read); + } + return output.toByteArray(); + } catch (IOException e) { + throw new SkillException("Failed to read Skill content stream.", e); + } + } + + private static void deleteTemporaryFile(Path path) { + if (path == null) { + return; + } + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + // 临时文件清理由操作系统兜底,调用方异常语义保持不变。 + } + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java new file mode 100644 index 0000000..7a39bdd --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java @@ -0,0 +1,633 @@ +package com.easyagents.skill.store.file; + +import com.easyagents.skill.exception.SkillException; +import com.easyagents.skill.store.SkillContentStage; +import com.easyagents.skill.store.SkillContentStore; +import com.easyagents.skill.util.SkillHashes; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.ref.Cleaner; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * 基于进程临时目录的流式 Skill 内容存储。 + * + *

内容先写入隔离暂存文件,提交后按 SHA-256 去重,并由引用计数控制物理文件生命周期。 + * 调用方应在不再使用内容引用和已打开的输入流后调用 {@link #close()}。未显式关闭的实例会由 + * JVM Cleaner 尝试兜底清理,但 Cleaner 不应替代正常的生命周期管理。

+ */ +public final class TemporaryFileSkillContentStore implements SkillContentStore, AutoCloseable { + + private static final String DIRECTORY_PREFIX = "easy-agents-skill-content-"; + private static final int COPY_BUFFER_SIZE = 16 * 1024; + private static final Cleaner CLEANER = Cleaner.create(); + + private final Object lifecycleMonitor = new Object(); + private final Path storageDirectory; + private final Path stagedDirectory; + private final Path contentDirectory; + private final Map stagedContents = new HashMap<>(); + private final Map contents = new HashMap<>(); + private final Cleaner.Cleanable cleanable; + private volatile boolean closed; + + /** + * 在系统临时目录中创建独立内容存储。 + */ + public TemporaryFileSkillContentStore() { + this(null); + } + + /** + * 在指定父目录中创建独立内容存储。 + * + *

该实例只拥有新建的子目录,关闭时不会删除传入的父目录。

+ * + * @param parentDirectory 临时存储父目录;为 null 时使用系统临时目录 + */ + public TemporaryFileSkillContentStore(Path parentDirectory) { + this.storageDirectory = createStorageDirectory(parentDirectory); + this.stagedDirectory = storageDirectory.resolve("staged"); + this.contentDirectory = storageDirectory.resolve("content"); + try { + Files.createDirectory(stagedDirectory); + Files.createDirectory(contentDirectory); + } catch (IOException exception) { + deleteDirectoryQuietly(storageDirectory); + throw new SkillException("Failed to initialize temporary Skill content store.", exception); + } + this.cleanable = CLEANER.register(this, new DirectoryCleanup(storageDirectory)); + } + + /** + * 流式保存字节内容并持有一个正式引用。 + * + * @param bytes 内容字节;null 按空内容处理 + * @return SHA-256 内容引用 + */ + @Override + public String put(byte[] bytes) { + byte[] safeBytes = bytes == null ? new byte[0] : bytes; + return put(new ByteArrayInputStream(safeBytes), safeBytes.length); + } + + /** + * 流式保存内容并持有一个正式引用。 + * + * @param inputStream 内容流,不由本方法关闭 + * @param maxBytes 最大允许字节数 + * @return SHA-256 内容引用 + */ + @Override + public String put(InputStream inputStream, long maxBytes) { + SkillContentStage stage = stage(inputStream, maxBytes); + try { + return commit(stage); + } catch (RuntimeException exception) { + try { + rollback(stage); + } catch (RuntimeException cleanupError) { + exception.addSuppressed(cleanupError); + } + throw exception; + } + } + + /** + * 将内容流式写入独立暂存文件并同步计算 SHA-256。 + * + * @param inputStream 内容流,不由本方法关闭 + * @param maxBytes 最大允许字节数 + * @return 暂存结果 + */ + @Override + public SkillContentStage stage(InputStream inputStream, long maxBytes) { + if (inputStream == null || maxBytes < 0) { + throw new SkillException("Valid Skill content stream and limit are required."); + } + String stageId = UUID.randomUUID().toString(); + Path stagedPath = stagedDirectory.resolve(stageId + ".stage"); + ensureOpen(); + try { + MessageDigest digest = SkillHashes.newSha256Digest(); + long size = 0; + try (DigestOutputStream output = new DigestOutputStream( + Files.newOutputStream(stagedPath, StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE), digest)) { + byte[] buffer = new byte[COPY_BUFFER_SIZE]; + int length; + while ((length = inputStream.read(buffer)) >= 0) { + if (length == 0) { + continue; + } + if (size > maxBytes - length) { + throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); + } + output.write(buffer, 0, length); + size += length; + } + } + String contentHash = SkillHashes.toHex(digest.digest()); + String contentRef = "sha256:" + contentHash; + StagedContent stagedContent = new StagedContent( + stagedPath, contentRef, contentHash, size); + synchronized (lifecycleMonitor) { + ensureOpenLocked(); + stagedContents.put(stageId, stagedContent); + } + return new SkillContentStage(stageId, contentRef, contentHash, size, false); + } catch (IOException | RuntimeException exception) { + deleteFileQuietly(stagedPath); + if (exception instanceof SkillException skillException) { + throw skillException; + } + throw new SkillException("Failed to stage temporary Skill content.", exception); + } + } + + /** + * 原子地提交暂存内容并增加一份正式引用。 + * + * @param stage 暂存结果 + * @return SHA-256 内容引用 + */ + @Override + public String commit(SkillContentStage stage) { + if (stage == null) { + throw new SkillException("Skill content stage is required."); + } + synchronized (lifecycleMonitor) { + ensureOpenLocked(); + StagedContent stagedContent = stagedContents.get(stage.getStageId()); + validateStage(stage, stagedContent); + validateStagedFile(stagedContent); + + StoredContent existing = contents.get(stagedContent.contentRef); + if (existing != null) { + validateDuplicateContent(stagedContent, existing); + if (existing.references == Long.MAX_VALUE) { + throw new SkillException("Skill content reference count overflow: " + + stagedContent.contentRef); + } + deleteFile(stagedContent.path, "Failed to delete committed Skill stage."); + stagedContents.remove(stage.getStageId()); + existing.references++; + existing.pendingDeletion = false; + return stagedContent.contentRef; + } + + Path targetPath = contentDirectory.resolve(stagedContent.contentHash + ".content"); + if (Files.exists(targetPath, LinkOption.NOFOLLOW_LINKS)) { + throw new SkillException("Unexpected Skill content file already exists: " + + stagedContent.contentRef); + } + moveAtomically(stagedContent.path, targetPath); + contents.put(stagedContent.contentRef, + new StoredContent(targetPath, stagedContent.size)); + stagedContents.remove(stage.getStageId()); + return stagedContent.contentRef; + } + } + + /** + * 删除尚未提交的暂存内容。 + * + * @param stage 暂存结果 + */ + @Override + public void rollback(SkillContentStage stage) { + if (stage == null) { + return; + } + synchronized (lifecycleMonitor) { + if (closed) { + return; + } + StagedContent stagedContent = stagedContents.get(stage.getStageId()); + if (stagedContent == null) { + return; + } + validateStage(stage, stagedContent); + deleteFile(stagedContent.path, "Failed to rollback staged Skill content."); + stagedContents.remove(stage.getStageId()); + } + } + + /** + * 增加一份正式内容引用。 + * + * @param contentRef 内容引用 + */ + @Override + public void retain(String contentRef) { + synchronized (lifecycleMonitor) { + ensureOpenLocked(); + StoredContent content = contents.get(contentRef); + if (content == null || content.references <= 0) { + throw new SkillException("Skill content does not exist: " + contentRef); + } + if (content.references == Long.MAX_VALUE) { + throw new SkillException("Skill content reference count overflow: " + contentRef); + } + content.references++; + } + } + + /** + * 释放一份正式内容引用,最后一个引用释放后删除物理文件。 + * + * @param contentRef 内容引用 + */ + @Override + public void release(String contentRef) { + if (contentRef == null || contentRef.isBlank()) { + return; + } + synchronized (lifecycleMonitor) { + if (closed) { + return; + } + StoredContent content = contents.get(contentRef); + if (content == null) { + return; + } + if (content.references > 0) { + content.references--; + } + if (content.references == 0) { + content.pendingDeletion = true; + deleteReleasedContent(contentRef, content); + } + } + } + + /** + * 打开正式内容的文件流。 + * + *

返回流会持有读取租约,最后一个正式引用释放时会等待已打开的流关闭后再删除文件。

+ * + * @param contentRef 内容引用 + * @return 内容输入流,调用方负责关闭 + */ + @Override + public InputStream open(String contentRef) { + synchronized (lifecycleMonitor) { + ensureOpenLocked(); + StoredContent content = contents.get(contentRef); + if (content == null || content.references <= 0) { + throw new SkillException("Skill content does not exist: " + contentRef); + } + if (content.readers == Integer.MAX_VALUE) { + throw new SkillException("Skill content reader count overflow: " + contentRef); + } + try { + InputStream input = Files.newInputStream(content.path, StandardOpenOption.READ); + content.readers++; + return new LeasedInputStream(input, contentRef, content); + } catch (IOException exception) { + throw new SkillException("Failed to open temporary Skill content: " + contentRef, + exception); + } + } + } + + /** + * 读取正式内容的全部字节。 + * + *

该兼容方法会按接口约定返回一个字节数组;流式调用方应优先使用 {@link #open(String)}。

+ * + * @param contentRef 内容引用 + * @return 内容字节 + */ + @Override + public byte[] readAllBytes(String contentRef) { + try (InputStream input = open(contentRef)) { + return input.readAllBytes(); + } catch (IOException exception) { + throw new SkillException("Failed to read temporary Skill content: " + contentRef, + exception); + } + } + + /** + * 判断正式内容是否仍有有效引用。 + * + * @param contentRef 内容引用 + * @return 存在且至少持有一个引用时为 true + */ + @Override + public boolean exists(String contentRef) { + synchronized (lifecycleMonitor) { + if (closed) { + return false; + } + StoredContent content = contents.get(contentRef); + return content != null && content.references > 0 + && Files.isRegularFile(content.path, LinkOption.NOFOLLOW_LINKS); + } + } + + /** + * 清理全部暂存和正式内容并关闭存储。 + * + *

调用前应先关闭通过 {@link #open(String)} 获取的流。该方法可重复调用。

+ */ + @Override + public void close() { + synchronized (lifecycleMonitor) { + if (closed) { + return; + } + closed = true; + stagedContents.clear(); + contents.clear(); + } + try { + deleteDirectory(storageDirectory); + } catch (IOException exception) { + throw new SkillException("Failed to clean temporary Skill content store.", exception); + } finally { + cleanable.clean(); + } + } + + /** + * 返回该实例拥有的临时目录,供同包生命周期测试使用。 + * + * @return 实例临时目录 + */ + Path storageDirectory() { + return storageDirectory; + } + + private void ensureOpen() { + synchronized (lifecycleMonitor) { + ensureOpenLocked(); + } + } + + private void ensureOpenLocked() { + if (closed) { + throw new SkillException("Temporary Skill content store is closed."); + } + } + + private static void validateStage(SkillContentStage stage, StagedContent content) { + if (content == null) { + throw new SkillException("Skill content stage does not exist: " + stage.getStageId()); + } + if (!content.contentRef.equals(stage.getContentRef()) + || !content.contentHash.equals(stage.getContentHash()) + || content.size != stage.getSize()) { + throw new SkillException("Skill content stage metadata does not match: " + + stage.getStageId()); + } + } + + private static void validateStagedFile(StagedContent content) { + try { + BasicFileAttributes attributes = Files.readAttributes( + content.path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!attributes.isRegularFile() || attributes.size() != content.size) { + throw new SkillException("Staged Skill content file changed before commit: " + + content.contentRef); + } + } catch (IOException exception) { + throw new SkillException("Failed to inspect staged Skill content: " + + content.contentRef, exception); + } + } + + private static void validateDuplicateContent(StagedContent stagedContent, + StoredContent existing) { + if (existing.size != stagedContent.size) { + throw new SkillException("SHA-256 collision detected for Skill content: " + + stagedContent.contentRef); + } + try { + if (Files.mismatch(stagedContent.path, existing.path) != -1) { + throw new SkillException("SHA-256 collision detected for Skill content: " + + stagedContent.contentRef); + } + } catch (IOException exception) { + throw new SkillException("Failed to compare duplicate Skill content: " + + stagedContent.contentRef, exception); + } + } + + private static void moveAtomically(Path source, Path target) { + try { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException exception) { + try { + Files.move(source, target); + } catch (IOException fallbackException) { + fallbackException.addSuppressed(exception); + throw new SkillException("Failed to commit staged Skill content.", + fallbackException); + } + } catch (IOException exception) { + throw new SkillException("Failed to commit staged Skill content.", exception); + } + } + + private void deleteReleasedContent(String contentRef, StoredContent content) { + if (content.readers > 0) { + return; + } + deleteFile(content.path, "Failed to delete released Skill content."); + contents.remove(contentRef, content); + } + + private void releaseReader(String contentRef, StoredContent content) throws IOException { + synchronized (lifecycleMonitor) { + if (content.readers > 0) { + content.readers--; + } + if (closed || !content.pendingDeletion || content.readers > 0) { + return; + } + try { + deleteReleasedContent(contentRef, content); + } catch (SkillException exception) { + throw new IOException("Failed to delete released Skill content.", exception); + } + } + } + + private static Path createStorageDirectory(Path parentDirectory) { + Path directory = null; + try { + directory = parentDirectory == null + ? Files.createTempDirectory(DIRECTORY_PREFIX) + : Files.createTempDirectory(parentDirectory, DIRECTORY_PREFIX); + restrictOwnerAccess(directory); + return directory; + } catch (IOException exception) { + deleteDirectoryQuietly(directory); + throw new SkillException("Failed to create temporary Skill content store.", exception); + } + } + + private static void restrictOwnerAccess(Path directory) throws IOException { + try { + Files.setPosixFilePermissions(directory, EnumSet.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE)); + } catch (UnsupportedOperationException ignored) { + // 非 POSIX 文件系统使用平台默认的临时目录访问控制。 + } + } + + private static void deleteFile(Path path, String message) { + try { + Files.deleteIfExists(path); + } catch (IOException exception) { + throw new SkillException(message, exception); + } + } + + private static void deleteFileQuietly(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + // 后续 close 或 Cleaner 会再次清理实例目录。 + } + } + + private static void deleteDirectory(Path directory) throws IOException { + if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS)) { + return; + } + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) + throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path current, IOException exception) + throws IOException { + if (exception != null) { + throw exception; + } + Files.deleteIfExists(current); + return FileVisitResult.CONTINUE; + } + }); + } + + private static void deleteDirectoryQuietly(Path directory) { + if (directory == null) { + return; + } + try { + deleteDirectory(directory); + } catch (IOException ignored) { + // Cleaner 是兜底路径,无法向已回收实例传播清理失败。 + } + } + + private final class LeasedInputStream extends FilterInputStream { + + private final String contentRef; + private final StoredContent content; + private boolean streamClosed; + + private LeasedInputStream(InputStream input, String contentRef, StoredContent content) { + super(input); + this.contentRef = contentRef; + this.content = content; + } + + @Override + public void close() throws IOException { + if (streamClosed) { + return; + } + streamClosed = true; + IOException closeFailure = null; + try { + super.close(); + } catch (IOException exception) { + closeFailure = exception; + } + try { + releaseReader(contentRef, content); + } catch (IOException cleanupFailure) { + if (closeFailure == null) { + closeFailure = cleanupFailure; + } else { + closeFailure.addSuppressed(cleanupFailure); + } + } + if (closeFailure != null) { + throw closeFailure; + } + } + } + + private static final class StagedContent { + + private final Path path; + private final String contentRef; + private final String contentHash; + private final long size; + + private StagedContent(Path path, String contentRef, String contentHash, long size) { + this.path = path; + this.contentRef = contentRef; + this.contentHash = contentHash; + this.size = size; + } + } + + private static final class StoredContent { + + private final Path path; + private final long size; + private long references = 1; + private int readers; + private boolean pendingDeletion; + + private StoredContent(Path path, long size) { + this.path = path; + this.size = size; + } + } + + private static final class DirectoryCleanup implements Runnable { + + private final Path directory; + + private DirectoryCleanup(Path directory) { + this.directory = directory; + } + + @Override + public void run() { + deleteDirectoryQuietly(directory); + } + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java index 98632aa..e33f711 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java @@ -1,24 +1,30 @@ package com.easyagents.skill.store.memory; import com.easyagents.skill.exception.SkillException; +import com.easyagents.skill.store.SkillContentStage; import com.easyagents.skill.store.SkillContentStore; import com.easyagents.skill.util.SkillHashes; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; /** - * 基于内存的 Skill 二进制内容存储。 + * 基于内存的 Skill 内容存储,支持真实暂存与引用计数,适用于测试和轻量场景。 */ public class InMemorySkillContentStore implements SkillContentStore { - private final ConcurrentMap contents = new ConcurrentHashMap<>(); + private final ConcurrentMap contents = new ConcurrentHashMap<>(); + private final ConcurrentMap stagedContents = new ConcurrentHashMap<>(); /** - * 保存内容并返回内容引用。 + * 保存内容并持有一个引用。 * * @param bytes 内容字节 * @return 内容引用 @@ -27,10 +33,109 @@ public class InMemorySkillContentStore implements SkillContentStore { public String put(byte[] bytes) { byte[] safeBytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); String contentRef = SkillHashes.sha256Ref(safeBytes); - contents.putIfAbsent(contentRef, safeBytes); + contents.compute(contentRef, (key, current) -> { + if (current == null) { + return new StoredContent(safeBytes); + } + if (!Arrays.equals(current.bytes, safeBytes)) { + throw new SkillException("SHA-256 collision detected for Skill content: " + contentRef); + } + current.references.incrementAndGet(); + return current; + }); return contentRef; } + /** + * 流式保存内容并持有一个引用。 + * + * @param inputStream 内容流 + * @param maxBytes 最大允许字节数 + * @return 内容引用 + */ + @Override + public String put(InputStream inputStream, long maxBytes) { + return put(readBounded(inputStream, maxBytes)); + } + + /** + * 将内容写入独立暂存区。 + * + * @param inputStream 内容流 + * @param maxBytes 最大允许字节数 + * @return 暂存结果 + */ + @Override + public SkillContentStage stage(InputStream inputStream, long maxBytes) { + byte[] bytes = readBounded(inputStream, maxBytes); + String hash = SkillHashes.sha256Hex(bytes); + String stageId = UUID.randomUUID().toString(); + stagedContents.put(stageId, bytes); + return new SkillContentStage(stageId, "sha256:" + hash, hash, bytes.length, false); + } + + /** + * 原子地将暂存内容转为正式引用。 + * + * @param stage 暂存结果 + * @return 正式内容引用 + */ + @Override + public String commit(SkillContentStage stage) { + if (stage == null) { + throw new SkillException("Skill content stage is required."); + } + byte[] bytes = stagedContents.remove(stage.getStageId()); + if (bytes == null) { + throw new SkillException("Skill content stage does not exist: " + stage.getStageId()); + } + String contentRef = put(bytes); + if (!contentRef.equals(stage.getContentRef())) { + release(contentRef); + throw new SkillException("Skill staged content hash changed before commit."); + } + return contentRef; + } + + /** + * 删除暂存内容。 + * + * @param stage 暂存结果 + */ + @Override + public void rollback(SkillContentStage stage) { + if (stage != null) { + stagedContents.remove(stage.getStageId()); + } + } + + /** + * 增加正式内容引用计数。 + * + * @param contentRef 内容引用 + */ + @Override + public void retain(String contentRef) { + contents.compute(contentRef, (key, content) -> { + if (content == null) { + throw new SkillException("Skill content does not exist: " + contentRef); + } + content.references.incrementAndGet(); + return content; + }); + } + + /** + * 释放正式内容引用并在归零时删除内容。 + * + * @param contentRef 内容引用 + */ + @Override + public void release(String contentRef) { + contents.computeIfPresent(contentRef, (key, content) -> + content.references.decrementAndGet() <= 0 ? null : content); + } + /** * 打开内容流。 * @@ -46,15 +151,15 @@ public class InMemorySkillContentStore implements SkillContentStore { * 读取全部内容。 * * @param contentRef 内容引用 - * @return 内容字节 + * @return 内容副本 */ @Override public byte[] readAllBytes(String contentRef) { - byte[] bytes = contents.get(contentRef); - if (bytes == null) { + StoredContent content = contents.get(contentRef); + if (content == null) { throw new SkillException("Skill content does not exist: " + contentRef); } - return Arrays.copyOf(bytes, bytes.length); + return Arrays.copyOf(content.bytes, content.bytes.length); } /** @@ -67,4 +172,39 @@ public class InMemorySkillContentStore implements SkillContentStore { public boolean exists(String contentRef) { return contents.containsKey(contentRef); } + + private static byte[] readBounded(InputStream inputStream, long maxBytes) { + if (inputStream == null || maxBytes < 0) { + throw new SkillException("Valid Skill content stream and limit are required."); + } + try { + ByteArrayOutputStream output = new ByteArrayOutputStream((int) Math.min(maxBytes, 8_192)); + byte[] buffer = new byte[8_192]; + long total = 0; + int read; + while ((read = inputStream.read(buffer)) >= 0) { + if (read == 0) { + continue; + } + total += read; + if (total > maxBytes) { + throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); + } + output.write(buffer, 0, read); + } + return output.toByteArray(); + } catch (IOException e) { + throw new SkillException("Failed to read Skill content stream.", e); + } + } + + private static final class StoredContent { + + private final byte[] bytes; + private final AtomicInteger references = new AtomicInteger(1); + + private StoredContent(byte[] bytes) { + this.bytes = Arrays.copyOf(bytes, bytes.length); + } + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java index 85e748e..f0e0335 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillFrontmatter.java @@ -1,89 +1,293 @@ package com.easyagents.skill.util; import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.model.SkillDocument; +import com.easyagents.skill.model.SkillPackageLimits; +import com.easyagents.skill.model.SkillSourceLocation; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.error.MarkedYAMLException; +import org.yaml.snakeyaml.error.YAMLException; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; +import java.io.StringReader; +import java.nio.charset.CharacterCodingException; +import java.util.ArrayList; +import java.util.Date; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TimeZone; /** - * SKILL.md frontmatter 解析工具。 + * SKILL.md 安全 frontmatter 解析与确定性序列化工具。 */ public final class SkillFrontmatter { + private static final String DOCUMENT_PATH = "SKILL.md"; + private static final Set ALLOWED_TAGS = Set.of( + Tag.MAP, Tag.SEQ, Tag.STR, Tag.BOOL, Tag.NULL, + Tag.INT, Tag.FLOAT, Tag.TIMESTAMP + ); + private SkillFrontmatter() { } /** - * 解析 SKILL.md 开头的 frontmatter。 + * 解析 SKILL.md frontmatter,并兼容原有必填字段校验行为。 * * @param content SKILL.md 原始内容 - * @return frontmatter 键值 + * @return frontmatter 有序键值 + * @throws SkillValidationException 文档或必填字段不合法 */ public static Map parse(String content) { - if (content == null || content.isBlank()) { - throw new SkillValidationException("SKILL.md content is required."); - } - String[] lines = content.split("\\R", -1); - if (lines.length == 0 || !"---".equals(lines[0])) { - throw new SkillValidationException("SKILL.md must start with frontmatter."); - } - - Map values = new LinkedHashMap<>(); - boolean closed = false; - for (int i = 1; i < lines.length; i++) { - String line = lines[i]; - if ("---".equals(line)) { - closed = true; - break; - } - if (line.isBlank()) { - continue; - } - if (Character.isWhitespace(line.charAt(0))) { - throw new SkillValidationException("Nested frontmatter is not supported."); - } - parseScalarLine(line, values); - } - if (!closed) { - throw new SkillValidationException("SKILL.md frontmatter is not closed."); - } - if (isBlank(values.get("name"))) { - throw new SkillValidationException("SKILL.md frontmatter name is required."); - } - if (isBlank(values.get("description"))) { - throw new SkillValidationException("SKILL.md frontmatter description is required."); - } + SkillDocument document = parseDocument(content, SkillPackageLimits.defaults()); + Map values = document.getFrontmatter().getValues(); + requireCoreField(values, "name"); + requireCoreField(values, "description"); return values; } - private static void parseScalarLine(String line, Map values) { - int colonIndex = line.indexOf(':'); - if (colonIndex <= 0) { - throw new SkillValidationException("Only single-line key: value frontmatter is supported."); - } - String key = line.substring(0, colonIndex).trim(); - String value = stripQuotes(line.substring(colonIndex + 1).trim()); - if (key.isBlank()) { - throw new SkillValidationException("Frontmatter key cannot be blank."); - } - if (value.isBlank()) { - throw new SkillValidationException("Frontmatter value cannot be blank: " + key); - } - values.put(key, value); + /** + * 使用默认安全限额解析完整 SKILL.md。 + * + * @param content SKILL.md 原始内容 + * @return 解析后的文档 + * @throws SkillValidationException 文档语法不合法 + */ + public static SkillDocument parseDocument(String content) { + return parseDocument(content, SkillPackageLimits.defaults()); } - private static String stripQuotes(String value) { - if (value.length() >= 2) { - boolean doubleQuoted = value.startsWith("\"") && value.endsWith("\""); - boolean singleQuoted = value.startsWith("'") && value.endsWith("'"); - if (doubleQuoted || singleQuoted) { - return value.substring(1, value.length() - 1); + /** + * 使用指定安全限额解析完整 SKILL.md。 + * + * @param content SKILL.md 原始内容 + * @param limits 安全限额 + * @return 解析后的文档 + * @throws SkillValidationException 文档语法、类型或限额不合法 + */ + public static SkillDocument parseDocument(String content, SkillPackageLimits limits) { + if (content == null || content.isBlank()) { + throw error("SKILL_CONTENT_REQUIRED", 1, 1, "SKILL.md content is required.", null); + } + try { + SkillUtf8.byteLength(content); + } catch (CharacterCodingException e) { + throw error("INVALID_UTF8", 1, 1, + "SKILL.md must be losslessly encodable as UTF-8.", e); + } + SkillPackageLimits effectiveLimits = limits == null ? SkillPackageLimits.defaults() : limits; + FrontmatterSections sections = split(content); + long frontmatterBytes; + try { + frontmatterBytes = SkillUtf8.byteLength(sections.yaml); + } catch (CharacterCodingException e) { + throw error("INVALID_UTF8", 2, 1, + "SKILL.md frontmatter must be losslessly encodable as UTF-8.", e); + } + if (frontmatterBytes > effectiveLimits.getMaxFrontmatterBytes()) { + throw error("FRONTMATTER_TOO_LARGE", 1, 1, + "SKILL.md frontmatter exceeds " + effectiveLimits.getMaxFrontmatterBytes() + " bytes.", null); + } + + LoaderOptions loaderOptions = new LoaderOptions(); + loaderOptions.setAllowDuplicateKeys(false); + loaderOptions.setWarnOnDuplicateKeys(false); + loaderOptions.setAllowRecursiveKeys(false); + loaderOptions.setMaxAliasesForCollections(effectiveLimits.getMaxYamlAliases()); + loaderOptions.setNestingDepthLimit(effectiveLimits.getMaxYamlDepth()); + loaderOptions.setCodePointLimit(effectiveLimits.getMaxYamlCodePoints()); + loaderOptions.setTagInspector(tag -> isStandardTag(tag)); + + try { + Yaml yaml = new Yaml(new SafeConstructor(loaderOptions)); + Node rootNode = yaml.compose(new StringReader(sections.yaml)); + Object loaded = yaml.load(sections.yaml); + if (!(loaded instanceof Map map)) { + throw error("FRONTMATTER_ROOT_NOT_MAP", 2, 1, + "SKILL.md frontmatter root must be a YAML mapping.", null); } + LinkedHashMap values = copyStringMap(map, 0, effectiveLimits.getMaxYamlDepth()); + return new SkillDocument(content, values, sections.body, + collectTopLevelLocations(rootNode), List.of()); + } catch (SkillValidationException e) { + throw e; + } catch (MarkedYAMLException e) { + int line = e.getProblemMark() == null ? 2 : e.getProblemMark().getLine() + 2; + int column = e.getProblemMark() == null ? 1 : e.getProblemMark().getColumn() + 1; + throw error("INVALID_FRONTMATTER_YAML", line, column, + "Invalid SKILL.md frontmatter: " + safeProblem(e), e); + } catch (YAMLException e) { + throw error("INVALID_FRONTMATTER_YAML", 2, 1, + "Invalid SKILL.md frontmatter: " + safeProblem(e), e); } - return value; } - private static boolean isBlank(Object value) { - return value == null || value.toString().isBlank(); + /** + * 将有序 frontmatter 与 Markdown 正文确定性序列化为 SKILL.md。 + * + * @param values frontmatter 有序键值 + * @param markdownBody Markdown 正文 + * @return 完整 SKILL.md + * @throws SkillValidationException frontmatter 包含不安全类型 + */ + public static String serialize(Map values, String markdownBody) { + LinkedHashMap safeValues = copyStringMap( + values == null ? Map.of() : values, 0, SkillPackageLimits.defaults().getMaxYamlDepth()); + DumperOptions dumperOptions = new DumperOptions(); + dumperOptions.setAllowUnicode(true); + dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + dumperOptions.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN); + dumperOptions.setIndent(2); + dumperOptions.setIndicatorIndent(0); + dumperOptions.setPrettyFlow(true); + dumperOptions.setSplitLines(false); + dumperOptions.setWidth(4_096); + dumperOptions.setLineBreak(DumperOptions.LineBreak.UNIX); + dumperOptions.setDereferenceAliases(true); + + LoaderOptions loaderOptions = new LoaderOptions(); + loaderOptions.setAllowDuplicateKeys(false); + Representer representer = new Representer(dumperOptions); + representer.setTimeZone(TimeZone.getTimeZone("UTC")); + Yaml yaml = new Yaml(new SafeConstructor(loaderOptions), representer, dumperOptions, loaderOptions); + String yamlContent = yaml.dump(safeValues); + String body = markdownBody == null ? "" : markdownBody; + return "---\n" + yamlContent + "---\n" + body; + } + + private static FrontmatterSections split(String content) { + int yamlStart; + if (content.startsWith("---\n")) { + yamlStart = 4; + } else if (content.startsWith("---\r\n")) { + yamlStart = 5; + } else { + throw error("FRONTMATTER_START_REQUIRED", 1, 1, + "SKILL.md must start with a frontmatter delimiter.", null); + } + + int lineStart = yamlStart; + while (lineStart <= content.length()) { + int lineEnd = content.indexOf('\n', lineStart); + int contentEnd = lineEnd < 0 ? content.length() : lineEnd; + String line = content.substring(lineStart, contentEnd); + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + } + if ("---".equals(line)) { + int bodyStart = lineEnd < 0 ? content.length() : lineEnd + 1; + return new FrontmatterSections(content.substring(yamlStart, lineStart), + content.substring(bodyStart)); + } + if (lineEnd < 0) { + break; + } + lineStart = lineEnd + 1; + } + throw error("FRONTMATTER_NOT_CLOSED", 1, 1, + "SKILL.md frontmatter is not closed.", null); + } + + private static LinkedHashMap copyStringMap(Map source, int depth, int maxDepth) { + if (depth > maxDepth) { + throw error("YAML_DEPTH_LIMIT", 2, 1, + "SKILL.md frontmatter exceeds the YAML nesting depth limit.", null); + } + LinkedHashMap target = new LinkedHashMap<>(); + for (Map.Entry entry : source.entrySet()) { + if (!(entry.getKey() instanceof String key) || key.isBlank()) { + throw error("INVALID_FRONTMATTER_KEY", 2, 1, + "SKILL.md frontmatter keys must be non-blank strings.", null); + } + target.put(key, copyValue(entry.getValue(), depth + 1, maxDepth)); + } + return target; + } + + private static Object copyValue(Object value, int depth, int maxDepth) { + if (value == null || value instanceof String || value instanceof Boolean || value instanceof Number) { + return value; + } + if (value instanceof Date date) { + return new Date(date.getTime()); + } + if (value instanceof Map map) { + return copyStringMap(map, depth, maxDepth); + } + if (value instanceof List list) { + if (depth > maxDepth) { + throw error("YAML_DEPTH_LIMIT", 2, 1, + "SKILL.md frontmatter exceeds the YAML nesting depth limit.", null); + } + List target = new ArrayList<>(list.size()); + for (Object item : list) { + target.add(copyValue(item, depth + 1, maxDepth)); + } + return target; + } + throw error("UNSUPPORTED_FRONTMATTER_TYPE", 2, 1, + "Unsupported SKILL.md frontmatter value type: " + value.getClass().getName(), null); + } + + private static boolean isStandardTag(Tag tag) { + return tag != null && ALLOWED_TAGS.contains(tag); + } + + /** + * 收集顶层 frontmatter key 在完整 SKILL.md 中的一基位置。 + * + * @param rootNode YAML 根节点 + * @return 按源码顺序排列的字段位置 + */ + private static Map collectTopLevelLocations(Node rootNode) { + LinkedHashMap locations = new LinkedHashMap<>(); + if (!(rootNode instanceof MappingNode mappingNode)) { + return locations; + } + for (NodeTuple tuple : mappingNode.getValue()) { + if (!(tuple.getKeyNode() instanceof ScalarNode scalarNode) + || scalarNode.getStartMark() == null) { + continue; + } + locations.putIfAbsent(scalarNode.getValue(), new SkillSourceLocation( + scalarNode.getStartMark().getLine() + 2, + scalarNode.getStartMark().getColumn() + 1)); + } + return locations; + } + + private static void requireCoreField(Map values, String key) { + Object value = values.get(key); + if (value == null || value.toString().isBlank()) { + throw error("MISSING_" + key.toUpperCase(), 2, 1, + "SKILL.md frontmatter " + key + " is required.", null); + } + } + + private static String safeProblem(Exception exception) { + String message = exception.getMessage(); + if (message == null || message.isBlank()) { + return exception.getClass().getSimpleName(); + } + int lineBreak = message.indexOf('\n'); + return lineBreak < 0 ? message : message.substring(0, lineBreak); + } + + private static SkillValidationException error(String code, Integer line, Integer column, + String message, Throwable cause) { + return new SkillValidationException(code, DOCUMENT_PATH, line, column, message, cause); + } + + private record FrontmatterSections(String yaml, String body) { } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java index cefd9e1..ac8013f 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillHashes.java @@ -32,16 +32,34 @@ public final class SkillHashes { * @return 十六进制 hash */ public static String sha256Hex(byte[] bytes) { + return toHex(newSha256Digest().digest(bytes == null ? new byte[0] : bytes)); + } + + /** + * 创建新的 SHA-256 摘要器。 + * + * @return SHA-256 摘要器 + */ + public static MessageDigest newSha256Digest() { try { - MessageDigest digest = MessageDigest.getInstance(SHA_256); - byte[] hashed = digest.digest(bytes == null ? new byte[0] : bytes); - StringBuilder builder = new StringBuilder(hashed.length * 2); - for (byte item : hashed) { - builder.append(String.format("%02x", item)); - } - return builder.toString(); + return MessageDigest.getInstance(SHA_256); } catch (NoSuchAlgorithmException e) { throw new SkillException("SHA-256 algorithm is unavailable.", e); } } + + /** + * 将字节转为小写十六进制文本。 + * + * @param bytes 原始字节 + * @return 小写十六进制文本 + */ + public static String toHex(byte[] bytes) { + StringBuilder builder = new StringBuilder(bytes.length * 2); + for (byte item : bytes) { + builder.append(Character.forDigit((item >>> 4) & 0x0F, 16)); + builder.append(Character.forDigit(item & 0x0F, 16)); + } + return builder.toString(); + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java index a2c61ba..6fe92e4 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java @@ -2,6 +2,7 @@ package com.easyagents.skill.util; import com.easyagents.skill.exception.SkillValidationException; +import java.text.Normalizer; import java.util.Locale; /** @@ -29,6 +30,11 @@ public final class SkillPaths { */ public static final String ASSETS_DIR = "assets"; + /** + * example 顶级目录。 + */ + public static final String EXAMPLES_DIR = "examples"; + private SkillPaths() { } @@ -42,17 +48,17 @@ public final class SkillPaths { if (path == null) { throw new SkillValidationException("Skill path is required."); } - String normalized = path.replace('\\', '/').trim(); - while (normalized.startsWith("./")) { - normalized = normalized.substring(2); + if (containsControlCharacter(path)) { + throw new SkillValidationException("Skill path contains a control character."); } + String normalized = Normalizer.normalize(path.replace('\\', '/'), Normalizer.Form.NFC); if (normalized.isEmpty()) { throw new SkillValidationException("Skill path cannot be empty."); } if (normalized.startsWith("/") || normalized.matches("^[A-Za-z]:.*")) { throw new SkillValidationException("Absolute skill path is not allowed: " + path); } - String[] segments = normalized.split("/"); + String[] segments = normalized.split("/", -1); for (String segment : segments) { if (segment.isEmpty() || ".".equals(segment) || "..".equals(segment)) { throw new SkillValidationException("Unsafe skill path is not allowed: " + path); @@ -60,10 +66,33 @@ public final class SkillPaths { if (segment.startsWith(".")) { throw new SkillValidationException("Hidden skill path is not allowed: " + path); } + if (!segment.equals(segment.strip())) { + throw new SkillValidationException("Skill path segments cannot have surrounding whitespace: " + path); + } } return normalized; } + /** + * 生成大小写不敏感的路径冲突键。 + * + * @param path 逻辑路径 + * @return 路径冲突键 + */ + public static String collisionKey(String path) { + return normalize(path).toLowerCase(Locale.ROOT); + } + + /** + * 获取路径段数量。 + * + * @param path 逻辑路径 + * @return 路径深度 + */ + public static int depth(String path) { + return normalize(path).split("/", -1).length; + } + /** * 获取一级目录或文件名。 * @@ -115,4 +144,15 @@ public final class SkillPaths { || normalized.endsWith("/.DS_Store") || ".DS_Store".equals(normalized); } + + private static boolean containsControlCharacter(String path) { + for (int index = 0; index < path.length(); index++) { + char character = path.charAt(index); + if (character <= 0x1F || character == 0x7F + || Character.getType(character) == Character.FORMAT) { + return true; + } + } + return false; + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java new file mode 100644 index 0000000..67f9ed9 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java @@ -0,0 +1,187 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillAsset; +import com.easyagents.skill.model.SkillMetadata; +import com.easyagents.skill.model.SkillReference; +import com.easyagents.skill.model.SkillResource; +import com.easyagents.skill.model.SkillResourceKind; +import com.easyagents.skill.model.SkillScript; +import com.easyagents.skill.model.SkillScriptLanguage; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * 通用 Skill 资源与旧资源视图之间的兼容适配工具。 + */ +public final class SkillResources { + + private static final Set TEXT_EXTENSIONS = Set.of( + ".md", ".markdown", ".txt", ".json", ".yaml", ".yml", ".xml", ".csv", ".tsv", + ".htm", ".html", ".css", ".properties", ".toml", ".ini", ".sql", ".java", ".kt", ".kts", + ".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".sh", ".bash", ".zsh", + ".rb", ".go", ".rs", ".c", ".h", ".cpp", ".hpp", ".gradle", ".groovy" + ); + + private SkillResources() { + } + + /** + * 根据顶层目录识别资源语义类型。 + * + * @param path Skill 根目录相对路径 + * @return 资源语义类型 + */ + public static SkillResourceKind classify(String path) { + String topDirectory = SkillPaths.firstSegment(path); + return switch (topDirectory) { + case SkillPaths.REFERENCES_DIR -> SkillResourceKind.REFERENCE; + case SkillPaths.SCRIPTS_DIR -> SkillResourceKind.SCRIPT; + case SkillPaths.ASSETS_DIR -> SkillResourceKind.ASSET; + case SkillPaths.EXAMPLES_DIR -> SkillResourceKind.EXAMPLE; + default -> SkillResourceKind.OTHER; + }; + } + + /** + * 判断资源是否应按严格 UTF-8 文本处理。 + * + * @param path 资源路径 + * @param kind 资源语义类型 + * @param mediaType 媒体类型 + * @return 文本资源时为 true + */ + public static boolean isText(String path, SkillResourceKind kind, String mediaType) { + if (kind == SkillResourceKind.SCRIPT) { + return true; + } + if (kind == SkillResourceKind.ASSET) { + return false; + } + if (mediaType != null && (mediaType.startsWith("text/") + || mediaType.contains("json") || mediaType.contains("yaml") + || mediaType.contains("xml") || mediaType.contains("javascript"))) { + return true; + } + String lowerPath = path.toLowerCase(Locale.ROOT); + return TEXT_EXTENSIONS.stream().anyMatch(lowerPath::endsWith); + } + + /** + * 获取 Skill 的正式通用资源;旧模型会被按需转换。 + * + * @param skill Skill 聚合 + * @return 通用资源副本 + */ + public static List canonicalResources(Skill skill) { + if (skill == null) { + return new ArrayList<>(); + } + return new ArrayList<>(skill.getResources()); + } + + /** + * 将尚未迁移的旧 references、scripts、assets 视图转换为正式通用资源。 + * + *

该方法只读取旧视图,不读取或修改正式资源列表,由 {@link Skill#getResources()} + * 在第一次正式访问时完成一次性迁移。

+ * + * @param skill Skill 聚合 + * @return 从旧视图转换得到的通用资源副本 + */ + public static List fromLegacyViews(Skill skill) { + List resources = new ArrayList<>(); + if (skill == null) { + return resources; + } + if (skill.getReferences() != null) { + for (SkillReference reference : skill.getReferences()) { + SkillResource resource = base(reference.getPath(), SkillResourceKind.REFERENCE, + "text/markdown", reference.getContentHash(), reference.getSize(), reference.getMetadata()); + resource.setTextContent(reference.getContent()); + resources.add(resource); + } + } + if (skill.getScripts() != null) { + for (SkillScript script : skill.getScripts()) { + SkillResource resource = base(script.getPath(), SkillResourceKind.SCRIPT, + "text/plain", script.getContentHash(), script.getSize(), script.getMetadata()); + resource.setTextContent(script.getContent()); + resources.add(resource); + } + } + if (skill.getAssets() != null) { + for (SkillAsset asset : skill.getAssets()) { + SkillResource resource = base(asset.getPath(), SkillResourceKind.ASSET, + asset.getMediaType(), asset.getContentHash(), asset.getSize(), asset.getMetadata()); + resource.setContentRef(asset.getContentRef()); + resources.add(resource); + } + } + return resources; + } + + /** + * 依据通用资源刷新旧 references、scripts、assets 兼容视图。 + * + * @param skill Skill 聚合 + */ + public static void refreshLegacyViews(Skill skill) { + List references = new ArrayList<>(); + List scripts = new ArrayList<>(); + List assets = new ArrayList<>(); + for (SkillResource resource : skill.getResources()) { + if (resource.getKind() == SkillResourceKind.REFERENCE && resource.isText()) { + SkillReference reference = new SkillReference(); + reference.setPath(resource.getPath()); + reference.setName(SkillPaths.fileName(resource.getPath())); + reference.setContent(resource.getTextContent()); + reference.setContentHash(resource.getContentHash()); + reference.setSize(resource.getSize()); + reference.setMetadata(copy(resource.getMetadata())); + references.add(reference); + } else if (resource.getKind() == SkillResourceKind.SCRIPT && resource.isText()) { + SkillScript script = new SkillScript(); + script.setPath(resource.getPath()); + script.setLanguage(SkillScriptLanguage.fromPath(resource.getPath())); + script.setContent(resource.getTextContent()); + script.setContentHash(resource.getContentHash()); + script.setSize(resource.getSize()); + script.setMetadata(copy(resource.getMetadata())); + scripts.add(script); + } else if (resource.getKind() == SkillResourceKind.ASSET) { + SkillAsset asset = new SkillAsset(); + asset.setPath(resource.getPath()); + asset.setName(SkillPaths.fileName(resource.getPath())); + asset.setMediaType(resource.getMediaType()); + asset.setContentRef(resource.getContentRef()); + asset.setContentHash(resource.getContentHash()); + asset.setSize(resource.getSize()); + asset.setMetadata(copy(resource.getMetadata())); + assets.add(asset); + } + } + skill.setReferences(references); + skill.setScripts(scripts); + skill.setAssets(assets); + } + + private static SkillResource base(String path, SkillResourceKind kind, String mediaType, + String hash, long size, SkillMetadata metadata) { + SkillResource resource = new SkillResource(); + resource.setPath(path); + resource.setKind(kind); + resource.setMediaType(mediaType); + resource.setContentHash(hash); + resource.setSize(size); + resource.setMetadata(copy(metadata)); + return resource; + } + + private static SkillMetadata copy(SkillMetadata metadata) { + return metadata == null ? new SkillMetadata() : new SkillMetadata(metadata.getValues()); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillUtf8.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillUtf8.java new file mode 100644 index 0000000..60ddda8 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillUtf8.java @@ -0,0 +1,56 @@ +package com.easyagents.skill.util; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** + * Skill 文本的严格 UTF-8 编码工具。 + */ +public final class SkillUtf8 { + + private SkillUtf8() { + } + + /** + * 严格编码文本,拒绝孤立 surrogate 等非法 UTF-16 输入。 + * + * @param text 待编码文本 + * @return UTF-8 字节 + * @throws CharacterCodingException 文本不能无损编码为 UTF-8 + */ + public static byte[] encode(String text) throws CharacterCodingException { + ByteBuffer encoded = newEncoder().encode(CharBuffer.wrap( + Objects.requireNonNull(text, "text"))); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + return bytes; + } + + /** + * 严格计算文本的 UTF-8 字节数。 + * + * @param text 待计算文本 + * @return UTF-8 字节数 + * @throws CharacterCodingException 文本不能无损编码为 UTF-8 + */ + public static long byteLength(String text) throws CharacterCodingException { + return newEncoder().encode(CharBuffer.wrap( + Objects.requireNonNull(text, "text"))).remaining(); + } + + /** + * 创建拒绝替换非法输入的 UTF-8 编码器。 + * + * @return 严格 UTF-8 编码器 + */ + private static CharsetEncoder newEncoder() { + return StandardCharsets.UTF_8.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationIssue.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationIssue.java new file mode 100644 index 0000000..44743c4 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationIssue.java @@ -0,0 +1,79 @@ +package com.easyagents.skill.validation; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Objects; + +/** + * 可定位的 Skill 校验问题。 + */ +public final class SkillValidationIssue implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private final String code; + private final SkillValidationSeverity severity; + private final String path; + private final Integer line; + private final Integer column; + private final String message; + private final String suggestion; + + /** + * 创建 Skill 校验问题。 + * + * @param code 稳定问题码 + * @param severity 严重级别 + * @param path 文件路径 + * @param line 一基行号 + * @param column 一基列号 + * @param message 问题说明 + * @param suggestion 修复建议 + */ + public SkillValidationIssue(String code, SkillValidationSeverity severity, String path, + Integer line, Integer column, String message, String suggestion) { + this.code = Objects.requireNonNull(code, "code"); + this.severity = Objects.requireNonNull(severity, "severity"); + this.path = path; + this.line = line; + this.column = column; + this.message = Objects.requireNonNull(message, "message"); + this.suggestion = suggestion; + } + + /** @return 稳定问题码 */ + public String getCode() { + return code; + } + + /** @return 严重级别 */ + public SkillValidationSeverity getSeverity() { + return severity; + } + + /** @return 文件路径 */ + public String getPath() { + return path; + } + + /** @return 一基行号 */ + public Integer getLine() { + return line; + } + + /** @return 一基列号 */ + public Integer getColumn() { + return column; + } + + /** @return 问题说明 */ + public String getMessage() { + return message; + } + + /** @return 修复建议 */ + public String getSuggestion() { + return suggestion; + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationMode.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationMode.java new file mode 100644 index 0000000..0e18798 --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationMode.java @@ -0,0 +1,17 @@ +package com.easyagents.skill.validation; + +/** + * Skill 标准校验模式。 + */ +public enum SkillValidationMode { + + /** + * 兼容历史包进入草稿;可修复的兼容问题以 warning 返回。 + */ + DRAFT_IMPORT, + + /** + * 正式新建、发布与标准导出;所有标准互操作约束均严格执行。 + */ + STANDARD +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationReport.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationReport.java new file mode 100644 index 0000000..1951fee --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationReport.java @@ -0,0 +1,89 @@ +package com.easyagents.skill.validation; + +import com.easyagents.skill.exception.SkillValidationException; + +import java.io.Serial; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Skill 结构化校验报告。 + */ +public final class SkillValidationReport implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private final List issues = new ArrayList<>(); + + /** + * 添加校验问题。 + * + * @param issue 校验问题 + * @return 当前报告 + */ + public SkillValidationReport add(SkillValidationIssue issue) { + if (issue != null) { + issues.add(issue); + } + return this; + } + + /** + * 合并其他报告。 + * + * @param report 其他报告 + * @return 当前报告 + */ + public SkillValidationReport merge(SkillValidationReport report) { + if (report != null) { + issues.addAll(report.issues); + } + return this; + } + + /** + * 获取不可变问题列表。 + * + * @return 问题列表 + */ + public List getIssues() { + return Collections.unmodifiableList(issues); + } + + /** + * 判断是否存在错误。 + * + * @return 存在 ERROR 时为 true + */ + public boolean hasErrors() { + return issues.stream().anyMatch(issue -> issue.getSeverity() == SkillValidationSeverity.ERROR); + } + + /** + * 判断报告是否为空。 + * + * @return 没有问题时为 true + */ + public boolean isEmpty() { + return issues.isEmpty(); + } + + /** + * 错误存在时抛出包含完整报告的异常。 + * + * @throws SkillValidationException 校验报告包含错误 + */ + public void throwIfInvalid() { + if (hasErrors()) { + SkillValidationIssue first = issues.stream() + .filter(issue -> issue.getSeverity() == SkillValidationSeverity.ERROR) + .findFirst() + .orElseThrow(); + throw new SkillValidationException(first.getCode(), first.getPath(), first.getLine(), + first.getColumn(), first.getMessage(), null, this); + } + } +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationSeverity.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationSeverity.java new file mode 100644 index 0000000..030fa1a --- /dev/null +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidationSeverity.java @@ -0,0 +1,16 @@ +package com.easyagents.skill.validation; + +/** + * Skill 校验问题严重级别。 + */ +public enum SkillValidationSeverity { + + /** 阻止导入确认、标准导出或发布。 */ + ERROR, + + /** 允许进入草稿但需要用户处理。 */ + WARNING, + + /** 不影响操作的信息提示。 */ + INFO +} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java index b0ef7dc..8654b40 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java @@ -1,6 +1,8 @@ package com.easyagents.skill.validation; +import com.easyagents.skill.exception.SkillValidationException; import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillPackageLimits; /** * Skill 校验接口。 @@ -13,4 +15,51 @@ public interface SkillValidator { * @param skill Skill 聚合 */ void validate(Skill skill); + + /** + * 聚合校验 Skill 并返回结构化报告。 + * + *

兼容默认实现会把旧命令式异常转换为单个错误;新实现应覆盖以返回全部问题。

+ * + * @param skill Skill 聚合 + * @return 结构化校验报告 + */ + default SkillValidationReport validateReport(Skill skill) { + SkillValidationReport report = new SkillValidationReport(); + try { + validate(skill); + } catch (SkillValidationException e) { + report.add(new SkillValidationIssue(e.getCode(), SkillValidationSeverity.ERROR, e.getPath(), + e.getLine(), e.getColumn(), e.getMessage(), null)); + } + return report; + } + + /** + * 使用当前 Codec 操作的安全限额聚合校验 Skill。 + * + *

兼容实现默认委托给原有校验入口;需要检查包限额的实现应覆盖本方法。

+ * + * @param skill Skill 聚合 + * @param limits 当前读写操作的安全限额 + * @return 结构化校验报告 + */ + default SkillValidationReport validateReport(Skill skill, SkillPackageLimits limits) { + return validateReport(skill); + } + + /** + * 使用指定标准模式和当前 Codec 安全限额聚合校验 Skill。 + * + *

兼容实现默认忽略模式;需要区分草稿导入与正式标准约束的实现应覆盖本方法。

+ * + * @param skill Skill 聚合 + * @param limits 当前读写操作的安全限额 + * @param mode 标准校验模式 + * @return 结构化校验报告 + */ + default SkillValidationReport validateReport(Skill skill, SkillPackageLimits limits, + SkillValidationMode mode) { + return validateReport(skill, limits); + } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java index 06fc481..7cf7ba7 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java @@ -1,156 +1,528 @@ package com.easyagents.skill.validation.defaults; import com.easyagents.skill.exception.SkillValidationException; -import com.easyagents.skill.model.*; +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillDocument; +import com.easyagents.skill.model.SkillPackageLimits; +import com.easyagents.skill.model.SkillResource; +import com.easyagents.skill.model.SkillResourceKind; +import com.easyagents.skill.model.SkillScriptLanguage; +import com.easyagents.skill.model.SkillSourceLocation; import com.easyagents.skill.util.SkillFrontmatter; import com.easyagents.skill.util.SkillHashes; import com.easyagents.skill.util.SkillPaths; +import com.easyagents.skill.util.SkillResources; +import com.easyagents.skill.util.SkillUtf8; +import com.easyagents.skill.validation.SkillValidationIssue; +import com.easyagents.skill.validation.SkillValidationMode; +import com.easyagents.skill.validation.SkillValidationReport; +import com.easyagents.skill.validation.SkillValidationSeverity; import com.easyagents.skill.validation.SkillValidator; -import java.nio.charset.StandardCharsets; +import java.nio.charset.CharacterCodingException; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; /** - * 默认 Skill 聚合校验器。 + * 默认 Skill 聚合结构化校验器。 */ public class DefaultSkillValidator implements SkillValidator { + private static final Pattern CANONICAL_NAME = Pattern.compile("[a-z0-9]+(?:-[a-z0-9]+)*"); + private static final Pattern LEGACY_UNDERSCORE_NAME = Pattern.compile("[a-z0-9]+(?:_[a-z0-9]+)+"); + private static final Pattern SHA_256 = Pattern.compile("[a-f0-9]{64}"); + + private final SkillPackageLimits limits; + /** - * 校验 Skill 聚合。 + * 使用默认安全限额创建校验器。 + */ + public DefaultSkillValidator() { + this(SkillPackageLimits.defaults()); + } + + /** + * 使用指定安全限额创建校验器。 + * + * @param limits 安全限额 + */ + public DefaultSkillValidator(SkillPackageLimits limits) { + this.limits = limits == null ? SkillPackageLimits.defaults() : limits; + } + + /** + * 校验 Skill 聚合,存在错误时抛出包含完整报告的异常。 * * @param skill Skill 聚合 + * @throws SkillValidationException 校验失败 */ @Override public void validate(Skill skill) { + validateReport(skill, limits, SkillValidationMode.STANDARD).throwIfInvalid(); + } + + /** + * 聚合校验 Skill。 + * + * @param skill Skill 聚合 + * @return 结构化校验报告 + */ + @Override + public SkillValidationReport validateReport(Skill skill) { + return validateReport(skill, limits); + } + + /** + * 使用当前 Codec 操作的安全限额聚合校验 Skill。 + * + * @param skill Skill 聚合 + * @param operationLimits 当前读写操作的安全限额 + * @return 结构化校验报告 + */ + @Override + public SkillValidationReport validateReport(Skill skill, SkillPackageLimits operationLimits) { + return validateReport(skill, operationLimits, SkillValidationMode.DRAFT_IMPORT); + } + + /** + * 使用指定标准模式和当前操作限额聚合校验 Skill。 + * + * @param skill Skill 聚合 + * @param operationLimits 当前读写操作的安全限额 + * @param mode 标准校验模式 + * @return 结构化校验报告 + */ + @Override + public SkillValidationReport validateReport(Skill skill, SkillPackageLimits operationLimits, + SkillValidationMode mode) { + SkillPackageLimits effectiveLimits = operationLimits == null ? limits : operationLimits; + SkillValidationMode effectiveMode = mode == null + ? SkillValidationMode.DRAFT_IMPORT : mode; + SkillValidationReport report = new SkillValidationReport(); if (skill == null) { - throw new SkillValidationException("Skill is required."); + return report.add(error("SKILL_REQUIRED", null, "Skill is required.", null)); } - requireText(skill.getId(), "Skill id is required."); - requireText(skill.getName(), "Skill name is required."); - requireText(skill.getDescription(), "Skill description is required."); - requireText(skill.getSkillContent(), "SKILL.md content is required."); - validateSkillFrontmatter(skill); - Set paths = new HashSet<>(); - paths.add(SkillPaths.SKILL_FILE); - validateReferences(skill.getReferences(), paths); - validateScripts(skill.getScripts(), paths); - validateAssets(skill.getAssets(), paths); + SkillDocument document = parseDocument(skill, report, effectiveLimits); + validateName(skill, document, report, effectiveMode); + validateDescription(skill, document, report); + if (document != null) { + validateDocument(skill, document, report); + } + List resources = SkillResources.canonicalResources(skill); + validateAggregateLimits(skill, resources, report, effectiveLimits); + validateResources(skill.getName(), resources, report, effectiveLimits); + return report; } - private static void validateReferences(List references, Set paths) { - if (references == null) { + private static void validateName(Skill skill, SkillDocument document, + SkillValidationReport report, SkillValidationMode mode) { + SkillSourceLocation location = sourceLocation(document, "name"); + String name = skill.getName(); + if (isBlank(name)) { + report.add(errorAt("NAME_REQUIRED", SkillPaths.SKILL_FILE, location, "Skill name is required.", + "Set frontmatter name to a portable Skill name.")); return; } - for (SkillReference reference : references) { - if (reference == null) { - throw new SkillValidationException("Skill reference cannot be null."); - } - String path = validateFilePath(reference.getPath(), SkillPaths.REFERENCES_DIR, paths); - if (!SkillPaths.hasExtension(path, ".md")) { - throw new SkillValidationException("Skill reference must be a markdown file: " + path); - } - requireContent(reference.getContent(), "Skill reference content is required: " + path); - validateTextHash(path, reference.getContent(), reference.getContentHash()); - validateSize(reference.getSize(), path); + if (name.length() > 64) { + report.add(errorAt("NAME_TOO_LONG", SkillPaths.SKILL_FILE, location, + "Skill name cannot exceed 64 characters.", "Shorten the frontmatter name.")); + } else if (LEGACY_UNDERSCORE_NAME.matcher(name).matches()) { + SkillValidationSeverity severity = mode == SkillValidationMode.STANDARD + ? SkillValidationSeverity.ERROR : SkillValidationSeverity.WARNING; + report.add(new SkillValidationIssue("NON_CANONICAL_NAME", severity, + SkillPaths.SKILL_FILE, line(location), column(location), + mode == SkillValidationMode.STANDARD + ? "Standard Skill name must use hyphens instead of underscores: " + name + : "Legacy underscore Skill name is accepted only for draft import: " + name, + "Replace underscores with single hyphens before publishing or standard export.")); + } else if (!CANONICAL_NAME.matcher(name).matches()) { + report.add(errorAt("INVALID_NAME", SkillPaths.SKILL_FILE, location, + "Skill name must use lowercase letters, numbers, and single hyphens.", + "Use a name such as data-analysis.")); + } + if (!isBlank(skill.getPackageRoot()) && !name.equals(skill.getPackageRoot())) { + report.add(errorAt("ROOT_NAME_MISMATCH", SkillPaths.SKILL_FILE, location, + "Skill package directory must match frontmatter name.", + "Rename the package directory or update frontmatter name.")); } } - private static void validateScripts(List scripts, Set paths) { - if (scripts == null) { - return; - } - for (SkillScript script : scripts) { - if (script == null) { - throw new SkillValidationException("Skill script cannot be null."); - } - String path = validateFilePath(script.getPath(), SkillPaths.SCRIPTS_DIR, paths); - if (SkillScriptLanguage.fromPath(path) == SkillScriptLanguage.UNKNOWN) { - throw new SkillValidationException("Unsupported skill script extension: " + path); - } - if (script.getLanguage() == SkillScriptLanguage.UNKNOWN - || script.getLanguage() != SkillScriptLanguage.fromPath(path)) { - throw new SkillValidationException("Skill script language does not match path: " + path); - } - requireContent(script.getContent(), "Skill script content is required: " + path); - validateTextHash(path, script.getContent(), script.getContentHash()); - validateSize(script.getSize(), path); + private static void validateDescription(Skill skill, SkillDocument document, + SkillValidationReport report) { + SkillSourceLocation location = sourceLocation(document, "description"); + if (isBlank(skill.getDescription())) { + report.add(errorAt("DESCRIPTION_REQUIRED", SkillPaths.SKILL_FILE, location, + "Skill description is required.", "Describe both the capability and when to use it.")); + } else if (skill.getDescription().length() > 1_024) { + report.add(errorAt("DESCRIPTION_TOO_LONG", SkillPaths.SKILL_FILE, location, + "Skill description cannot exceed 1024 characters.", "Shorten the description.")); } } - private static void validateAssets(List assets, Set paths) { - if (assets == null) { - return; + private static SkillDocument parseDocument(Skill skill, SkillValidationReport report, + SkillPackageLimits limits) { + if (isBlank(skill.getSkillContent())) { + report.add(error("SKILL_CONTENT_REQUIRED", SkillPaths.SKILL_FILE, + "SKILL.md content is required.", null)); + return null; } - for (SkillAsset asset : assets) { - if (asset == null) { - throw new SkillValidationException("Skill asset cannot be null."); - } - String path = validateFilePath(asset.getPath(), SkillPaths.ASSETS_DIR, paths); - requireText(asset.getName(), "Skill asset name is required: " + path); - requireText(asset.getMediaType(), "Skill asset media type is required: " + path); - requireText(asset.getContentRef(), "Skill asset content ref is required: " + path); - requireText(asset.getContentHash(), "Skill asset content hash is required: " + path); - if (!asset.getContentRef().equals("sha256:" + asset.getContentHash())) { - throw new SkillValidationException("Skill asset content ref does not match hash: " + path); - } - validateSize(asset.getSize(), path); + try { + return SkillFrontmatter.parseDocument(skill.getSkillContent(), limits); + } catch (SkillValidationException e) { + report.add(new SkillValidationIssue(e.getCode(), SkillValidationSeverity.ERROR, e.getPath(), + e.getLine(), e.getColumn(), e.getMessage(), "Fix the YAML frontmatter and retry.")); + return null; } } - private static void validateSkillFrontmatter(Skill skill) { - Map values = SkillFrontmatter.parse(skill.getSkillContent()); - if (!skill.getName().equals(values.get("name").toString())) { - throw new SkillValidationException("Skill name must match SKILL.md frontmatter."); - } - if (!skill.getDescription().equals(values.get("description").toString())) { - throw new SkillValidationException("Skill description must match SKILL.md frontmatter."); + private static void validateDocument(Skill skill, SkillDocument document, SkillValidationReport report) { + Map values = document.getFrontmatter().getValues(); + validateCoreString(document, values, "name", skill.getName(), report); + validateCoreString(document, values, "description", skill.getDescription(), report); + validateOptionalString(document, values, "license", null, report); + Object compatibility = values.get("compatibility"); + if (compatibility != null && (!(compatibility instanceof String text) + || text.isBlank() || text.length() > 500)) { + report.add(errorAt("INVALID_COMPATIBILITY", SkillPaths.SKILL_FILE, + sourceLocation(document, "compatibility"), + "Frontmatter compatibility must be a non-blank string of at most 500 characters.", null)); } + validateOptionalString(document, values, "allowed-tools", "INVALID_ALLOWED_TOOLS", report); + validateMetadataField(document, values.get("metadata"), report); if (skill.getMetadata() == null || !skill.getMetadata().getValues().equals(values)) { - throw new SkillValidationException("Skill metadata must match SKILL.md frontmatter."); + report.add(errorAt("METADATA_MISMATCH", SkillPaths.SKILL_FILE, + sourceLocation(document, "name"), + "Skill metadata must match SKILL.md frontmatter.", + "Reparse SKILL.md before saving the aggregate.")); + } + if (document.getMarkdownBody().length() > 30_000) { + report.add(new SkillValidationIssue("LONG_SKILL_BODY", SkillValidationSeverity.WARNING, + SkillPaths.SKILL_FILE, null, null, + "SKILL.md body is large and may reduce progressive-disclosure efficiency.", + "Move detailed material into references/.")); } } - private static String validateFilePath(String path, String expectedTopDir, Set paths) { - String normalized = SkillPaths.normalize(path); - if (!expectedTopDir.equals(SkillPaths.firstSegment(normalized))) { - throw new SkillValidationException("Skill file must be under " + expectedTopDir + "/: " + normalized); - } - if (normalized.indexOf('/') < 0 || normalized.endsWith("/")) { - throw new SkillValidationException("Skill file path must include file name: " + normalized); - } - if (!paths.add(normalized)) { - throw new SkillValidationException("Duplicate skill file path: " + normalized); - } - return normalized; - } - - private static void validateTextHash(String path, String content, String contentHash) { - requireText(contentHash, "Skill file content hash is required: " + path); - String actualHash = SkillHashes.sha256Hex(content.getBytes(StandardCharsets.UTF_8)); - if (!actualHash.equals(contentHash)) { - throw new SkillValidationException("Skill file content hash does not match: " + path); + private static void validateCoreString(SkillDocument document, Map values, + String key, String expected, + SkillValidationReport report) { + Object value = values.get(key); + if (!(value instanceof String text) || text.isBlank()) { + report.add(errorAt(key.toUpperCase(Locale.ROOT) + "_REQUIRED", SkillPaths.SKILL_FILE, + sourceLocation(document, key), + "SKILL.md frontmatter " + key + " must be a non-blank string.", null)); + } else if (!text.equals(expected)) { + report.add(errorAt(key.toUpperCase(Locale.ROOT) + "_MISMATCH", SkillPaths.SKILL_FILE, + sourceLocation(document, key), + "Skill " + key + " must match SKILL.md frontmatter.", null)); } } - private static void validateSize(long size, String path) { - if (size < 0) { - throw new SkillValidationException("Skill file size cannot be negative: " + path); + private static void validateOptionalString(SkillDocument document, Map values, + String key, String code, + SkillValidationReport report) { + Object value = values.get(key); + if (value != null && (!(value instanceof String text) || text.isBlank())) { + report.add(errorAt(code == null ? "INVALID_" + key.toUpperCase(Locale.ROOT) : code, + SkillPaths.SKILL_FILE, sourceLocation(document, key), + "Frontmatter " + key + " must be a non-blank string when provided.", null)); } } - private static void requireText(String value, String message) { - if (value == null || value.isBlank()) { - throw new SkillValidationException(message); + private static void validateMetadataField(SkillDocument document, Object metadata, + SkillValidationReport report) { + if (metadata == null) { + return; + } + if (!(metadata instanceof Map map)) { + report.add(errorAt("INVALID_METADATA", SkillPaths.SKILL_FILE, + sourceLocation(document, "metadata"), + "Frontmatter metadata must be a mapping.", null)); + return; + } + if (map.values().stream().anyMatch(value -> !(value instanceof String))) { + SkillSourceLocation location = sourceLocation(document, "metadata"); + report.add(new SkillValidationIssue("NON_STANDARD_METADATA", + SkillValidationSeverity.WARNING, SkillPaths.SKILL_FILE, + line(location), column(location), + "Nested or non-string metadata is preserved for AgentScope compatibility.", + "Use string values only for strict Agent Skills interoperability.")); } } - private static void requireContent(String value, String message) { - if (value == null) { - throw new SkillValidationException(message); + private static void validateResources(String skillName, List resources, + SkillValidationReport report, + SkillPackageLimits limits) { + Set exactPaths = new HashSet<>(); + Map collisionPaths = new HashMap<>(); + Map descendantPaths = new HashMap<>(); + exactPaths.add(SkillPaths.SKILL_FILE); + collisionPaths.put(SkillPaths.collisionKey(SkillPaths.SKILL_FILE), SkillPaths.SKILL_FILE); + for (SkillResource resource : resources) { + if (resource == null) { + report.add(error("NULL_RESOURCE", null, "Skill resource cannot be null.", null)); + continue; + } + if (isBlank(resource.getPath())) { + report.add(error("RESOURCE_PATH_REQUIRED", null, + "Skill resource path is required.", + "Set a safe relative path inside the Skill package.")); + continue; + } + try { + SkillUtf8.encode(resource.getPath()); + } catch (CharacterCodingException e) { + report.add(error("INVALID_UTF8_PATH", resource.getPath(), + "Skill resource path must be losslessly encodable as UTF-8.", + "Remove malformed UTF-16 surrogate code units from the path.")); + continue; + } + String path; + try { + path = SkillPaths.normalize(resource.getPath()); + } catch (SkillValidationException e) { + report.add(error("UNSAFE_RESOURCE_PATH", resource.getPath(), e.getMessage(), null)); + continue; + } + String portablePath = isBlank(skillName) ? path : skillName + "/" + path; + if (portablePath.length() > limits.getMaxPathLength()) { + report.add(error("PATH_LENGTH_LIMIT", path, + "Skill resource path exceeds the configured length limit.", null)); + } + if (SkillPaths.depth(portablePath) > limits.getMaxPathDepth()) { + report.add(error("PATH_DEPTH_LIMIT", path, + "Skill resource path exceeds the configured depth limit.", null)); + } + if (!path.equals(resource.getPath())) { + report.add(error("NON_NORMALIZED_PATH", path, + "Skill resource path must already be normalized.", null)); + } + if (!exactPaths.add(path)) { + report.add(error("DUPLICATE_RESOURCE_PATH", path, + "Duplicate Skill resource path: " + path, null)); + continue; + } + String collisionKey = SkillPaths.collisionKey(path); + String conflicting = collisionPaths.get(collisionKey); + if (conflicting != null) { + report.add(error("RESOURCE_PATH_COLLISION", path, + "Skill resource path conflicts with " + conflicting + ".", null)); + continue; + } + String hierarchyConflict = findHierarchyConflict(collisionKey, collisionPaths, descendantPaths); + if (hierarchyConflict != null) { + report.add(error("RESOURCE_PATH_HIERARCHY_CONFLICT", path, + "Skill resource path conflicts with file " + hierarchyConflict + + " in the same path hierarchy.", null)); + continue; + } + collisionPaths.put(collisionKey, path); + indexDescendantPath(collisionKey, path, descendantPaths); + SkillResourceKind expectedKind = SkillResources.classify(path); + if (resource.getKind() != expectedKind) { + report.add(error("RESOURCE_KIND_MISMATCH", path, + "Skill resource kind does not match its path.", null)); + } + validateResourceContent(resource, path, report, limits); } } + + private static void validateResourceContent(SkillResource resource, String path, + SkillValidationReport report, SkillPackageLimits limits) { + boolean expectedText = SkillResources.isText(path, resource.getKind(), resource.getMediaType()); + if (expectedText != resource.isText()) { + String code = resource.getKind() == SkillResourceKind.SCRIPT + ? "SCRIPT_TEXT_REQUIRED" : "RESOURCE_CONTENT_MODE_MISMATCH"; + report.add(error(code, path, + expectedText + ? "Resource path and media type require strict UTF-8 text content." + : "Resource path and media type require a binary content reference.", + "Store the resource using the canonical text or binary representation.")); + } + if (resource.getKind() == SkillResourceKind.SCRIPT && resource.isText() && resource.getSize() == 0) { + report.add(new SkillValidationIssue("EMPTY_SCRIPT", SkillValidationSeverity.WARNING, + path, null, null, "Script resource is empty.", + "Add script source or remove the unused file.")); + } + if (resource.getKind() == SkillResourceKind.SCRIPT + && SkillScriptLanguage.fromPath(path) == SkillScriptLanguage.UNKNOWN) { + report.add(new SkillValidationIssue("SCRIPT_LANGUAGE_UNRECOGNIZED", + SkillValidationSeverity.WARNING, path, null, null, + "Script language is not recognized from its extension.", + "Use .py, .js, or .sh for first-class editing and syntax highlighting.")); + } + if (isBlank(resource.getMediaType())) { + report.add(error("MEDIA_TYPE_REQUIRED", path, "Skill resource media type is required.", null)); + } + if (resource.getSize() < 0) { + report.add(error("NEGATIVE_RESOURCE_SIZE", path, + "Skill resource size cannot be negative.", null)); + } + byte[] textBytes = null; + if (resource.isText()) { + try { + textBytes = SkillUtf8.encode(resource.getTextContent()); + } catch (CharacterCodingException e) { + report.add(error("INVALID_UTF8", path, + "Text Skill resource must be losslessly encodable as UTF-8.", + "Remove malformed UTF-16 surrogate code units.")); + return; + } + } + if (isBlank(resource.getContentHash()) || !SHA_256.matcher(resource.getContentHash()).matches()) { + report.add(error("INVALID_RESOURCE_HASH", path, + "Skill resource SHA-256 is required and must be lowercase hexadecimal.", null)); + return; + } + if (resource.isText()) { + if (resource.getSize() > limits.getMaxTextFileBytes()) { + report.add(error("TEXT_FILE_SIZE_LIMIT", path, + "Text Skill resource exceeds the configured size limit.", null)); + } + if (resource.getContentRef() != null) { + report.add(error("AMBIGUOUS_RESOURCE_CONTENT", path, + "Text resources cannot also contain a binary content reference.", null)); + } + if (resource.getSize() != textBytes.length) { + report.add(error("RESOURCE_SIZE_MISMATCH", path, + "Skill resource size does not match UTF-8 content.", null)); + } + if (!resource.getContentHash().equals(SkillHashes.sha256Hex(textBytes))) { + report.add(error("RESOURCE_HASH_MISMATCH", path, + "Skill resource hash does not match text content.", null)); + } + } else { + if (resource.getSize() > limits.getMaxBinaryFileBytes()) { + report.add(error("BINARY_FILE_SIZE_LIMIT", path, + "Binary Skill resource exceeds the configured size limit.", null)); + } + if (isBlank(resource.getContentRef())) { + report.add(error("CONTENT_REF_REQUIRED", path, + "Binary Skill resource content reference is required.", null)); + } else if (!resource.getContentRef().equals("sha256:" + resource.getContentHash())) { + report.add(error("CONTENT_REF_MISMATCH", path, + "Binary Skill resource content reference must match its SHA-256.", null)); + } + } + } + + private static void validateAggregateLimits(Skill skill, List resources, + SkillValidationReport report, SkillPackageLimits limits) { + if (!isBlank(skill.getName())) { + String skillPath = skill.getName() + "/" + SkillPaths.SKILL_FILE; + if (skillPath.length() > limits.getMaxPathLength()) { + report.add(error("PATH_LENGTH_LIMIT", skillPath, + "Skill entry path exceeds the configured length limit.", null)); + } + if (SkillPaths.depth(skillPath) > limits.getMaxPathDepth()) { + report.add(error("PATH_DEPTH_LIMIT", skillPath, + "Skill entry path exceeds the configured depth limit.", null)); + } + } + if (resources.size() + 1 > limits.getMaxEntryCount()) { + report.add(error("ENTRY_COUNT_LIMIT", null, + "Skill contains too many files.", null)); + } + long total = 0; + if (skill.getSkillContent() != null) { + try { + total = SkillUtf8.byteLength(skill.getSkillContent()); + } catch (CharacterCodingException ignored) { + // parseDocument 已返回可定位的 INVALID_UTF8;非法文本不参与后续大小计算。 + } + } + if (total > limits.getMaxTextFileBytes()) { + report.add(error("TEXT_FILE_SIZE_LIMIT", SkillPaths.SKILL_FILE, + "SKILL.md exceeds the configured size limit.", null)); + } + for (SkillResource resource : resources) { + if (resource == null || resource.getSize() < 0) { + continue; + } + try { + total = Math.addExact(total, resource.getSize()); + } catch (ArithmeticException e) { + total = Long.MAX_VALUE; + } + if (total > limits.getMaxTotalUncompressedBytes()) { + report.add(error("TOTAL_SIZE_LIMIT", resource.getPath(), + "Skill resources exceed the configured total size limit.", null)); + break; + } + } + } + + /** + * 查找当前文件路径与已索引文件之间的祖先或后代冲突。 + * + * @param pathKey 当前路径冲突键 + * @param filePaths 已索引的文件路径 + * @param descendantPaths 已索引路径对应的首个后代文件 + * @return 冲突文件路径,不存在时返回 null + */ + private static String findHierarchyConflict(String pathKey, Map filePaths, + Map descendantPaths) { + String descendant = descendantPaths.get(pathKey); + if (descendant != null) { + return descendant; + } + int separator = pathKey.indexOf('/'); + while (separator >= 0) { + String ancestor = filePaths.get(pathKey.substring(0, separator)); + if (ancestor != null) { + return ancestor; + } + separator = pathKey.indexOf('/', separator + 1); + } + return null; + } + + /** + * 为文件路径的每个祖先建立后代索引,以常量时间识别反向插入冲突。 + * + * @param pathKey 文件路径冲突键 + * @param path 原始文件路径 + * @param descendantPaths 后代索引 + */ + private static void indexDescendantPath(String pathKey, String path, + Map descendantPaths) { + int separator = pathKey.indexOf('/'); + while (separator >= 0) { + descendantPaths.putIfAbsent(pathKey.substring(0, separator), path); + separator = pathKey.indexOf('/', separator + 1); + } + } + + private static SkillValidationIssue error(String code, String path, String message, String suggestion) { + return new SkillValidationIssue(code, SkillValidationSeverity.ERROR, path, null, null, + message, suggestion); + } + + private static SkillValidationIssue errorAt(String code, String path, SkillSourceLocation location, + String message, String suggestion) { + return new SkillValidationIssue(code, SkillValidationSeverity.ERROR, path, + line(location), column(location), message, suggestion); + } + + private static SkillSourceLocation sourceLocation(SkillDocument document, String key) { + return document == null ? null : document.getFrontmatterLocation(key); + } + + private static Integer line(SkillSourceLocation location) { + return location == null ? null : location.getLine(); + } + + private static Integer column(SkillSourceLocation location) { + return location == null ? null : location.getColumn(); + } + + private static boolean isBlank(String value) { + return value == null || value.isBlank(); + } } diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java index 50831bc..68b9dee 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java @@ -1,240 +1,1044 @@ package com.easyagents.skill.codec; import com.easyagents.skill.exception.SkillPackageException; +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.factory.SkillFactory; import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillPackage; +import com.easyagents.skill.model.SkillPackageLayout; +import com.easyagents.skill.model.SkillPackageLimits; +import com.easyagents.skill.model.SkillResource; +import com.easyagents.skill.model.SkillResourceKind; +import com.easyagents.skill.store.SkillContentStage; import com.easyagents.skill.store.SkillContentStore; import com.easyagents.skill.store.memory.InMemorySkillContentStore; +import com.easyagents.skill.util.SkillHashes; +import com.easyagents.skill.util.SkillPaths; +import com.easyagents.skill.validation.SkillValidator; +import org.apache.commons.compress.archivers.zip.UnixStat; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; /** - * ZipSkillPackageCodec 单元测试。 + * ZipSkillPackageCodec 安全边界与 round-trip 测试。 */ public class ZipSkillPackageCodecTest { /** - * 导入包含一个 Skill 文件夹的 zip。 + * 导出计划只保留已有文本源,不能再次持有整包文本 byte 数组。 + * + * @throws ClassNotFoundException 内部输出计划类型缺失 */ @Test - public void importOneSkillFolder() { - List skills = importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/references/rules/a.md", "# Rule", - "skill-a/scripts/run.py", "print('ok')", - "skill-a/assets/images/logo.png", "png-data" - )); + public void outputPlanDoesNotRetainTextByteArrays() throws ClassNotFoundException { + Class outputFile = Class.forName(ZipSkillPackageCodec.class.getName() + "$OutputFile"); - Assert.assertEquals(1, skills.size()); - Skill skill = skills.get(0); - Assert.assertEquals("skill-a", skill.getId()); - Assert.assertEquals("Skill A", skill.getName()); - Assert.assertEquals("Desc A", skill.getDescription()); + Assert.assertFalse(Arrays.stream(outputFile.getDeclaredFields()) + .anyMatch(field -> field.getType() == byte[].class)); + } + + /** + * 非 ZIP 输入与带任意前导数据的归档被拒绝。 + */ + @Test + public void rejectInvalidZipSignature() { + assertPackageCode("INVALID_ZIP", utf8("not-a-zip"), SkillPackageReadOptions.defaults()); + byte[] zip = zip(files("skill-a/SKILL.md", utf8(skillMd("skill-a")))); + byte[] withPreamble = new byte[zip.length + 1]; + withPreamble[0] = 1; + System.arraycopy(zip, 0, withPreamble, 1, zip.length); + assertPackageCode("INVALID_ZIP", withPreamble, SkillPackageReadOptions.defaults()); + } + + /** + * 单目录包导入通用资源并保持旧资源视图。 + */ + @Test + public void decodeWrappedSingleSkillWithGenericResources() { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + SkillPackageReadResult result = decode(new ZipSkillPackageCodec(store), zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/rules.md", utf8("# Rules"), + "skill-a/scripts/run.rb", utf8("puts 'ok'"), + "skill-a/examples/sample.json", utf8("{\"ok\":true}"), + "skill-a/docs/notes.txt", utf8("notes"), + "skill-a/assets/logo.bin", new byte[]{0, 1, 2} + ))); + + Assert.assertEquals(SkillPackageLayout.SINGLE_DIRECTORY, + result.getSkillPackage().getLayout()); + Skill skill = result.getSkillPackage().getSkills().get(0); + Assert.assertNull(skill.getId()); + Assert.assertEquals("skill-a", skill.getPackageRoot()); + Assert.assertEquals(5, skill.getResources().size()); + Assert.assertTrue(skill.getResources().stream().anyMatch(resource -> + resource.getKind() == SkillResourceKind.EXAMPLE)); + Assert.assertTrue(skill.getResources().stream().anyMatch(resource -> + resource.getKind() == SkillResourceKind.OTHER)); Assert.assertEquals(1, skill.getReferences().size()); - Assert.assertEquals("references/rules/a.md", skill.getReferences().get(0).getPath()); Assert.assertEquals(1, skill.getScripts().size()); Assert.assertEquals(1, skill.getAssets().size()); - Assert.assertEquals("assets/images/logo.png", skill.getAssets().get(0).getPath()); + Assert.assertTrue(store.exists(skill.getAssets().get(0).getContentRef())); } /** - * 导入包含多个 Skill 文件夹的 zip。 + * 根目录直压单 Skill 被正确识别。 */ @Test - public void importMultipleSkillFolders() { - List skills = importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-b/SKILL.md", skillMd("Skill B", "Desc B") - )); + public void decodeRootSkillPackage() { + SkillPackageReadResult result = decode(new ZipSkillPackageCodec(), zip(files( + "SKILL.md", utf8(skillMd("root-skill")), + "references/a.md", utf8("# A") + ))); - Assert.assertEquals(2, skills.size()); - Assert.assertEquals("skill-a", skills.get(0).getId()); - Assert.assertEquals("skill-b", skills.get(1).getId()); + Assert.assertEquals(SkillPackageLayout.ROOT_SKILL, result.getSkillPackage().getLayout()); + Assert.assertEquals("root-skill", result.getSkillPackage().getSkills().get(0).getPackageRoot()); } /** - * 拒绝 zip 根目录直接包含 SKILL.md。 - */ - @Test(expected = SkillPackageException.class) - public void rejectRootSkillMd() { - importZip(files("SKILL.md", skillMd("Skill", "Desc"))); - } - - /** - * 拒绝未知顶级目录。 - */ - @Test(expected = SkillPackageException.class) - public void rejectUnknownTopLevelDirectory() { - importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/unknown/a.md", "# Unknown" - )); - } - - /** - * 拒绝 references 下的非 md 文件。 - */ - @Test(expected = SkillPackageException.class) - public void rejectNonMarkdownReference() { - importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/references/a.txt", "text" - )); - } - - /** - * 拒绝不支持的脚本扩展名。 - */ - @Test(expected = SkillPackageException.class) - public void rejectUnsupportedScript() { - importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/scripts/run.rb", "puts 'ok'" - )); - } - - /** - * assets 允许任意扩展名。 + * 多目录包按稳定目录顺序导入。 */ @Test - public void allowArbitraryAssetExtension() { - List skills = importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/assets/model.custom", "asset" + public void decodeMultipleSkills() { + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + SkillPackageReadResult result = decode(codec, zip(files( + "skill-b/SKILL.md", utf8(skillMd("skill-b")), + "skill-a/SKILL.md", utf8(skillMd("skill-a")) + ))); + + Assert.assertEquals(SkillPackageLayout.MULTI_DIRECTORY, result.getSkillPackage().getLayout()); + Assert.assertEquals(List.of("skill-a", "skill-b"), result.getSkillPackage().getSkills().stream() + .map(Skill::getName).toList()); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + SkillPackageWriteResult writeResult = codec.encode( + result.getSkillPackage(), output, SkillPackageWriteOptions.defaults()); + Assert.assertEquals(SkillPackageLayout.MULTI_DIRECTORY, writeResult.getLayout()); + Assert.assertEquals(List.of("skill-a", "skill-b"), decode(codec, output.toByteArray()) + .getSkillPackage().getSkills().stream().map(Skill::getName).toList()); + } + + /** + * 嵌套 frontmatter 与未知字段在 Codec 中保留。 + */ + @Test + public void preserveNestedFrontmatter() { + String markdown = "---\nname: nested-skill\ndescription: Nested metadata\n" + + "metadata:\n owner: team-a\n levels:\n - 1\n - 2\n---\n# Nested\n"; + + Skill skill = decode(new ZipSkillPackageCodec(), zip(files( + "nested-skill/SKILL.md", utf8(markdown) + ))).getSkillPackage().getSkills().get(0); + + Assert.assertTrue(skill.getMetadata().get("metadata") instanceof Map); + Assert.assertEquals(markdown, skill.getSkillContent()); + } + + /** + * 根 SKILL.md 与包装 Skill 目录混用时拒绝。 + */ + @Test + public void rejectMixedPackageLayout() { + assertPackageCode("MIXED_PACKAGE_LAYOUT", zip(files( + "SKILL.md", utf8(skillMd("root-skill")), + "other-skill/SKILL.md", utf8(skillMd("other-skill")) + )), SkillPackageReadOptions.defaults()); + } + + /** + * 包装布局中的根散落文件被拒绝。 + */ + @Test + public void rejectUnownedRootFile() { + assertPackageCode("UNOWNED_ROOT_FILE", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "README.md", utf8("orphan") + )), SkillPackageReadOptions.defaults()); + } + + /** + * 路径穿越被拒绝。 + */ + @Test + public void rejectPathTraversal() { + assertPackageCode("UNSAFE_ENTRY_PATH", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/../escape.txt", utf8("escape") + )), SkillPackageReadOptions.defaults()); + } + + /** + * 大小写冲突路径被拒绝。 + */ + @Test + public void rejectCaseConflictingPaths() { + assertPackageCode("PATH_COLLISION", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/A.md", utf8("A"), + "skill-a/references/a.md", utf8("B") + )), SkillPackageReadOptions.defaults()); + } + + /** + * 普通文件不能同时作为另一个文件的祖先路径,且不受 entry 顺序影响。 + */ + @Test + public void rejectFileAndDescendantPathConflictsInBothOrders() { + assertPackageCode("PATH_HIERARCHY_CONFLICT", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references", utf8("file"), + "skill-a/references/a.md", utf8("descendant") + )), SkillPackageReadOptions.defaults()); + assertPackageCode("PATH_HIERARCHY_CONFLICT", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", utf8("descendant"), + "skill-a/references", utf8("file") + )), SkillPackageReadOptions.defaults()); + } + + /** + * Unicode NFC 等价路径被拒绝。 + */ + @Test + public void rejectUnicodeEquivalentPaths() { + assertPackageCode("DUPLICATE_ENTRY", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/\u00e9.md", utf8("A"), + "skill-a/references/e\u0301.md", utf8("B") + )), SkillPackageReadOptions.defaults()); + } + + /** + * 非法 UTF-8 文本被拒绝并指出文件路径。 + */ + @Test + public void rejectInvalidUtf8() { + byte[] bytes = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", new byte[]{(byte) 0xC3, 0x28} + )); + try { + decode(new ZipSkillPackageCodec(), bytes); + Assert.fail("Invalid UTF-8 should fail."); + } catch (SkillPackageException e) { + Assert.assertEquals("INVALID_UTF8", e.getCode()); + Assert.assertEquals("skill-a/references/a.md", e.getPath()); + } + } + + /** + * ZIP entry 原始文件名字节不是严格 UTF-8 时拒绝导入。 + */ + @Test + public void rejectInvalidUtf8EntryName() { + assertPackageCode("INVALID_UTF8_ENTRY_NAME", invalidUtf8EntryNameZip(), + SkillPackageReadOptions.defaults()); + } + + /** + * 中央目录 CRC 与实际内容不一致时拒绝。 + */ + @Test + public void rejectCrcMismatch() { + byte[] original = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")) )); - Assert.assertEquals(1, skills.get(0).getAssets().size()); - Assert.assertEquals("application/octet-stream", skills.get(0).getAssets().get(0).getMediaType()); + assertPackageCode("CRC_MISMATCH", tamperFirstCentralDirectoryCrc(original), + SkillPackageReadOptions.defaults()); } /** - * 忽略 macOS 系统文件和空目录。 + * Unix 符号链接 entry 被拒绝。 */ @Test - public void ignoreSystemFilesAndEmptyDirectories() { - Map files = files( - "__MACOSX/._x", "ignored", - "skill-a/.DS_Store", "ignored", - "skill-a/SKILL.md", skillMd("Skill A", "Desc A") - ); - List skills = new ZipSkillPackageCodec().importZip(new ByteArrayInputStream(zip(files, "skill-a/references/"))); - - Assert.assertEquals(1, skills.size()); - Assert.assertEquals("skill-a", skills.get(0).getId()); + public void rejectUnixSymlink() { + assertPackageCode("SYMLINK_ENTRY", symlinkZip(), SkillPackageReadOptions.defaults()); } /** - * 嵌套 references 和 assets 路径正常导入。 + * 目录和待忽略系统 entry 也必须先通过路径安全校验。 */ @Test - public void allowNestedManagedDirectories() { - List skills = importZip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/references/rules/a.md", "# Nested", - "skill-a/assets/images/logo.png", "logo" + public void rejectUnsafeDirectoryBeforeIgnoringEntries() { + assertPackageCode("UNSAFE_ENTRY_PATH", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")) + ), "../../"), SkillPackageReadOptions.defaults()); + assertPackageCode("UNSAFE_ENTRY_PATH", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "__MACOSX/../../escape", utf8("escape") + )), SkillPackageReadOptions.defaults()); + } + + /** + * entry 数、文本大小和压缩比限额均在读取前生效。 + */ + @Test + public void enforcePackageLimits() { + byte[] twoEntries = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", utf8("A") + )); + assertPackageCode("ENTRY_COUNT_LIMIT", twoEntries, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxEntryCount(1).build())); + + byte[] largeText = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", utf8("x".repeat(300)) + )); + assertPackageCode("TEXT_FILE_SIZE_LIMIT", largeText, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxTextFileBytes(128).build())); + + byte[] compressible = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/bomb.md", utf8("a".repeat(20_000)) + )); + assertPackageCode("COMPRESSION_RATIO_LIMIT", compressible, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxCompressionRatio(2D).build())); + } + + /** + * 中央目录预扫描不能信任 EOCD 的较小声明值,且必须在构造 ZipFile 前拒绝超量条目。 + */ + @Test + public void preflightCountsActualCentralDirectoryEntries() { + assertPackageCode("ENTRY_COUNT_LIMIT", syntheticCentralDirectoryZip(3, 1), + new SkillPackageReadOptions( + SkillPackageLimits.builder().maxEntryCount(2).build())); + } + + /** + * 未超限时,EOCD 声明数量与中央目录实际数量不一致必须作为非法 ZIP 拒绝。 + */ + @Test + public void rejectInconsistentCentralDirectoryEntryCount() { + assertPackageCode("INVALID_ZIP", syntheticCentralDirectoryZip(3, 1), + new SkillPackageReadOptions( + SkillPackageLimits.builder().maxEntryCount(4).build())); + } + + /** + * ZIP64 中央目录也必须按实际条目数在 ZipFile 对象化前执行上限。 + */ + @Test + public void preflightCountsActualZip64CentralDirectoryEntries() { + assertPackageCode("ENTRY_COUNT_LIMIT", syntheticZip64CentralDirectoryZip(3, 1), + new SkillPackageReadOptions( + SkillPackageLimits.builder().maxEntryCount(2).build())); + } + + /** + * 中央目录末尾被截断时,即使 EOCD 边界同步缩短也必须拒绝。 + */ + @Test + public void rejectTruncatedCentralDirectory() { + byte[] truncated = truncateLastCentralDirectoryByte( + syntheticCentralDirectoryZip(3, 3)); + + assertPackageCode("INVALID_ZIP", truncated, + new SkillPackageReadOptions( + SkillPackageLimits.builder().maxEntryCount(4).build())); + } + + /** + * 总量、二进制、路径深度、路径长度和压缩输入限制均生效。 + */ + @Test + public void enforceAdditionalPackageLimits() { + byte[] totalSize = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", utf8("x".repeat(120)) + )); + assertPackageCode("TOTAL_SIZE_LIMIT", totalSize, new SkillPackageReadOptions( + SkillPackageLimits.builder() + .maxTextFileBytes(128) + .maxBinaryFileBytes(128) + .maxTotalUncompressedBytes(180) + .build())); + + byte[] binarySize = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/a.bin", new byte[]{1, 2, 3} + )); + assertPackageCode("BINARY_FILE_SIZE_LIMIT", binarySize, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxBinaryFileBytes(2).build())); + + byte[] deepPath = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/nested/a.md", utf8("A") + )); + assertPackageCode("PATH_DEPTH_LIMIT", deepPath, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxPathDepth(2).build())); + + byte[] longPath = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/this-name-is-far-too-long.md", utf8("A") + )); + assertPackageCode("PATH_LENGTH_LIMIT", longPath, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxPathLength(30).build())); + + byte[] compressedInput = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")) + )); + assertPackageCode("COMPRESSED_SIZE_LIMIT", compressedInput, new SkillPackageReadOptions( + SkillPackageLimits.builder().maxCompressedPackageBytes(64).build())); + } + + /** + * 后续校验失败时兼容存储也不会提前提交资产。 + */ + @Test + public void rollbackStagedContentOnFailure() { + TrackingContentStore store = new TrackingContentStore(0); + byte[] bytes = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/data.bin", utf8("abc"), + "skill-b/SKILL.md", utf8(skillMd("Invalid Name")) )); - Assert.assertEquals("references/rules/a.md", skills.get(0).getReferences().get(0).getPath()); - Assert.assertEquals("assets/images/logo.png", skills.get(0).getAssets().get(0).getPath()); + try { + decode(new ZipSkillPackageCodec(store), bytes); + Assert.fail("Invalid multi-Skill package should fail."); + } catch (SkillPackageException expected) { + Assert.assertEquals(1, store.stageCalls); + Assert.assertEquals(0, store.stagedCount); + Assert.assertEquals(0, store.committedCount); + } } /** - * 资产字节写入内容存储。 + * 只读预检对有效二进制资源也只返回哈希身份,并回滚暂存内容。 */ @Test - public void storeAssetContent() { + public void reportOnlyRollsBackValidBinaryContentWithoutCommit() { + TrackingContentStore store = new TrackingContentStore(0); + SkillPackageReadResult result = new ZipSkillPackageCodec(store).decode( + new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/data.bin", new byte[]{1, 2, 3} + ))), SkillPackageReadOptions.reportOnly()); + + SkillResource resource = result.getSkillPackage().getSkills().get(0).getResources().get(0); + Assert.assertFalse(result.getValidationReport().hasErrors()); + Assert.assertEquals(1, store.stageCalls); + Assert.assertEquals(1, store.rollbackCalls); + Assert.assertEquals(0, store.commitAttempts); + Assert.assertEquals(0, store.stagedCount); + Assert.assertEquals(0, store.committedCount); + Assert.assertFalse(store.exists(resource.getContentRef())); + } + + /** + * 只读预检对可解析的无效包返回完整报告,并回滚二进制暂存内容。 + */ + @Test + public void reportOnlyReturnsValidationErrorsAndRollsBackBinaryContent() { + TrackingContentStore store = new TrackingContentStore(0); + SkillPackageReadResult result = new ZipSkillPackageCodec(store).decode( + new ByteArrayInputStream(zip(files( + "invalid-skill/SKILL.md", utf8(skillMd("Invalid Name")), + "invalid-skill/assets/data.bin", new byte[]{1, 2, 3} + ))), new SkillPackageReadOptions( + SkillPackageLimits.defaults(), SkillPackageReadMode.REPORT_ONLY)); + + Assert.assertTrue(result.getValidationReport().hasErrors()); + Assert.assertTrue(result.getValidationReport().getIssues().stream().anyMatch(issue -> + "INVALID_NAME".equals(issue.getCode()))); + Assert.assertEquals(1, store.stageCalls); + Assert.assertEquals(1, store.rollbackCalls); + Assert.assertEquals(0, store.commitAttempts); + Assert.assertEquals(0, store.stagedCount); + Assert.assertEquals(0, store.committedCount); + } + + /** + * 多 Skill 预检报告为每个相对路径补齐各自根目录。 + */ + @Test + public void reportOnlyPrefixesValidationPathsForMultipleSkills() { + SkillPackageReadResult result = new ZipSkillPackageCodec().decode( + new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", utf8(invalidDescriptionSkillMd("skill-a")), + "skill-b/SKILL.md", utf8(invalidDescriptionSkillMd("skill-b")) + ))), SkillPackageReadOptions.reportOnly()); + + Assert.assertTrue(result.getValidationReport().getIssues().stream().anyMatch(issue -> + "DESCRIPTION_REQUIRED".equals(issue.getCode()) + && "skill-a/SKILL.md".equals(issue.getPath()))); + Assert.assertTrue(result.getValidationReport().getIssues().stream().anyMatch(issue -> + "DESCRIPTION_REQUIRED".equals(issue.getCode()) + && "skill-b/SKILL.md".equals(issue.getPath()))); + } + + /** + * 多 Skill 标准导出的聚合错误也必须包含各自输出根目录。 + */ + @Test + public void exportPrefixesValidationPathsForMultipleSkills() { + Skill first = SkillFactory.create("first", skillMd("skill-a")); + first.setPackageRoot("skill-a"); + first.setDescription(null); + Skill second = SkillFactory.create("second", skillMd("skill-b")); + second.setPackageRoot("skill-b"); + second.setDescription(null); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try { + new ZipSkillPackageCodec().encode( + new SkillPackage(SkillPackageLayout.MULTI_DIRECTORY, + List.of(first, second)), + output, SkillPackageWriteOptions.defaults()); + Assert.fail("Invalid multi-Skill export should fail."); + } catch (SkillPackageException exception) { + Assert.assertTrue(exception.getReport().getIssues().stream().anyMatch(issue -> + "DESCRIPTION_REQUIRED".equals(issue.getCode()) + && "skill-a/SKILL.md".equals(issue.getPath()))); + Assert.assertTrue(exception.getReport().getIssues().stream().anyMatch(issue -> + "DESCRIPTION_REQUIRED".equals(issue.getCode()) + && "skill-b/SKILL.md".equals(issue.getPath()))); + Assert.assertEquals(0, output.size()); + } + } + + /** + * 包装 Skill 的 YAML 解析错误也必须返回完整包路径。 + */ + @Test + public void prefixWrappedSkillPathForMalformedYaml() { + try { + new ZipSkillPackageCodec().decode(new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", utf8( + "---\nname: skill-a\ndescription: [\n---\n# Invalid\n") + ))), SkillPackageReadOptions.reportOnly()); + Assert.fail("Malformed wrapped SKILL.md should fail."); + } catch (SkillPackageException exception) { + Assert.assertNotNull(exception.getReport()); + Assert.assertTrue(exception.getReport().getIssues().stream().anyMatch(issue -> + "skill-a/SKILL.md".equals(issue.getPath()))); + } + } + + /** + * 只读预检仍拒绝路径穿越等不可解析的结构安全错误。 + */ + @Test + public void reportOnlyStillRejectsStructuralSecurityErrors() { + assertPackageCode("UNSAFE_ENTRY_PATH", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/../escape.txt", utf8("escape") + )), SkillPackageReadOptions.reportOnly()); + } + + /** + * 部分 commit 失败时释放已提交引用并回滚其余暂存内容。 + */ + @Test + public void compensatePartiallyCommittedContent() { + TrackingContentStore store = new TrackingContentStore(2); + byte[] bytes = zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/a.bin", utf8("a"), + "skill-a/assets/b.bin", utf8("b") + )); + + try { + decode(new ZipSkillPackageCodec(store), bytes); + Assert.fail("Second commit should fail."); + } catch (SkillPackageException expected) { + Assert.assertEquals(2, store.stageCalls); + Assert.assertEquals(0, store.stagedCount); + Assert.assertEquals(0, store.committedCount); + } + } + + /** + * 全包校验成功后统一提交资产。 + */ + @Test + public void commitStagedContentAfterValidation() { + CountingContentStore store = new CountingContentStore(); + + decode(new ZipSkillPackageCodec(store), zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/data.bin", utf8("abc") + ))); + + Assert.assertEquals(1, store.putCount); + } + + /** + * 编码输出稳定,且 decode-encode-decode 语义等价。 + */ + @Test + public void stableEncodeAndSemanticRoundTrip() { InMemorySkillContentStore store = new InMemorySkillContentStore(); - List skills = new ZipSkillPackageCodec(store).importZip(new ByteArrayInputStream(zip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/assets/data.bin", "abc" - )))); + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(store); + SkillPackageReadResult first = decode(codec, zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", utf8("# A"), + "skill-a/assets/data.bin", new byte[]{1, 2, 3} + ))); + first.getSkillPackage().getSkills().get(0).setId("internal-repository-id"); - String contentRef = skills.get(0).getAssets().get(0).getContentRef(); - Assert.assertTrue(store.exists(contentRef)); - Assert.assertArrayEquals("abc".getBytes(StandardCharsets.UTF_8), store.readAllBytes(contentRef)); + ByteArrayOutputStream firstZip = new ByteArrayOutputStream(); + SkillPackageWriteResult firstWrite = codec.encode(first.getSkillPackage(), firstZip, + SkillPackageWriteOptions.defaults()); + ByteArrayOutputStream secondZip = new ByteArrayOutputStream(); + SkillPackageWriteResult secondWrite = codec.encode(first.getSkillPackage(), secondZip, + SkillPackageWriteOptions.defaults()); + + Assert.assertArrayEquals(firstZip.toByteArray(), secondZip.toByteArray()); + Assert.assertEquals(firstWrite.getPackageHash(), secondWrite.getPackageHash()); + Assert.assertEquals(SkillPackageLayout.SINGLE_DIRECTORY, firstWrite.getLayout()); + Assert.assertEquals(SkillHashes.sha256Hex(firstZip.toByteArray()), firstWrite.getPackageHash()); + Assert.assertFalse(zipContains(firstZip.toByteArray(), "internal-repository-id")); + + SkillPackageReadResult roundTrip = decode(codec, firstZip.toByteArray()); + Skill original = first.getSkillPackage().getSkills().get(0); + Skill decoded = roundTrip.getSkillPackage().getSkills().get(0); + Assert.assertNull(decoded.getId()); + Assert.assertEquals(original.getMetadata().getValues(), decoded.getMetadata().getValues()); + Assert.assertEquals(original.getSkillContent(), decoded.getSkillContent()); + Assert.assertEquals(original.getResources().stream().map(resource -> resource.getPath()).toList(), + decoded.getResources().stream().map(resource -> resource.getPath()).toList()); + Assert.assertEquals(original.getResources().stream().map(resource -> resource.getContentHash()).toList(), + decoded.getResources().stream().map(resource -> resource.getContentHash()).toList()); + Assert.assertTrue(allEntriesHaveStableTimestamp(firstZip.toByteArray())); } /** - * 导入失败时不写入资产内容。 + * Codec 对高压缩比合法文本自适应使用 STORED,保证同一限额下可回读。 */ @Test - public void failedImportDoesNotWriteAssetContent() { - CountingContentStore store = new CountingContentStore(); + public void highCompressionTextRoundTripsWithSameLimits() { + String content = skillMd("skill-a") + "a".repeat(20_000); + Skill skill = SkillFactory.create("id", content); + SkillPackage skillPackage = new SkillPackage( + SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)); + SkillPackageLimits limits = SkillPackageLimits.builder() + .maxCompressionRatio(2D) + .build(); + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + codec.encode(skillPackage, output, new SkillPackageWriteOptions(limits)); + SkillPackageReadResult decoded = codec.decode( + new ByteArrayInputStream(output.toByteArray()), new SkillPackageReadOptions(limits)); + + Assert.assertEquals(ZipEntry.STORED, zipEntryMethod(output.toByteArray(), "skill-a/SKILL.md")); + Assert.assertEquals(content, decoded.getSkillPackage().getSkills().get(0).getSkillContent()); + } + + /** + * 读写 options 的自定义宽松限额必须贯穿默认结构化校验器。 + */ + @Test + public void customOptionsLimitsReachValidatorDuringDecodeAndEncode() { + String longPath = "skill-a/references/" + "a".repeat(520) + ".md"; + SkillPackageLimits limits = SkillPackageLimits.builder() + .maxPathLength(600) + .build(); + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + SkillPackageReadResult imported = codec.decode( + new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + longPath, utf8("A") + ))), new SkillPackageReadOptions(limits)); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + codec.encode(imported.getSkillPackage(), output, new SkillPackageWriteOptions(limits)); + SkillPackageReadResult roundTrip = codec.decode( + new ByteArrayInputStream(output.toByteArray()), new SkillPackageReadOptions(limits)); + + Assert.assertEquals(longPath.substring("skill-a/".length()), + roundTrip.getSkillPackage().getSkills().get(0).getResources().get(0).getPath()); + } + + /** + * 写出端必须对自动生成的 Skill 根目录和 SKILL.md 路径应用调用方限额。 + */ + @Test + public void enforceOutputLimitsOnGeneratedSkillEntryPath() { + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + Skill skill = SkillFactory.create("id", skillMd("skill-a")); + String skillPath = "skill-a/SKILL.md"; + + assertExportIssue(codec, skill, new SkillPackageWriteOptions( + SkillPackageLimits.builder().maxPathLength(5).build()), + "PATH_LENGTH_LIMIT", skillPath); + assertExportIssue(codec, skill, new SkillPackageWriteOptions( + SkillPackageLimits.builder().maxPathDepth(1).build()), + "PATH_DEPTH_LIMIT", skillPath); + } + + /** + * 删除最后一个正式资源后不得从滞留的旧兼容视图恢复并再次导出。 + */ + @Test + public void emptyCanonicalResourcesDoNotFallBackToLegacyViews() { + String content = "# Rules"; + SkillResource reference = new SkillResource(); + reference.setPath("references/rules.md"); + reference.setKind(SkillResourceKind.REFERENCE); + reference.setMediaType("text/markdown"); + reference.setTextContent(content); + reference.setContentHash(SkillHashes.sha256Hex(utf8(content))); + reference.setSize(utf8(content).length); + Skill skill = SkillFactory.createWithResources( + "id", skillMd("skill-a"), List.of(reference)); + Assert.assertEquals(1, skill.getReferences().size()); + skill.getResources().clear(); + + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)), + output, SkillPackageWriteOptions.defaults()); + Skill decoded = codec.decode(new ByteArrayInputStream(output.toByteArray()), + SkillPackageReadOptions.defaults()).getSkillPackage().getSkills().get(0); + + Assert.assertTrue(decoded.getResources().isEmpty()); + Assert.assertTrue(decoded.getReferences().isEmpty()); + } + + /** + * 注入的默认校验器仍属于附加校验,其更严格限额不得被类型判断或操作选项绕过。 + */ + @Test + public void injectedDefaultValidatorAppliesItsStricterLimits() { + ZipSkillPackageCodec codec = new ZipSkillPackageCodec( + new InMemorySkillContentStore(), + new com.easyagents.skill.validation.defaults.DefaultSkillValidator( + SkillPackageLimits.builder().maxTextFileBytes(128).build())); + Skill skill = SkillFactory.create("id", skillMd("skill-a") + "x".repeat(300)); + + assertExportIssue(codec, skill, SkillPackageWriteOptions.defaults(), + "TEXT_FILE_SIZE_LIMIT", SkillPaths.SKILL_FILE); + } + + /** + * 自定义限额感知校验器必须收到当前 Codec 操作选项。 + */ + @Test + public void customValidatorReceivesOperationLimits() { + SkillPackageLimits limits = SkillPackageLimits.builder().maxPathLength(64).build(); + SkillPackageLimits[] observed = new SkillPackageLimits[1]; + SkillValidator validator = new SkillValidator() { + @Override + public void validate(Skill skill) { + // 限额感知实现通过下方结构化入口完成业务校验。 + } + + @Override + public com.easyagents.skill.validation.SkillValidationReport validateReport( + Skill skill, SkillPackageLimits operationLimits) { + observed[0] = operationLimits; + return new com.easyagents.skill.validation.SkillValidationReport(); + } + }; + ZipSkillPackageCodec codec = new ZipSkillPackageCodec( + new InMemorySkillContentStore(), validator); + + codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, + List.of(SkillFactory.create("id", skillMd("skill-a")))), + new ByteArrayOutputStream(), new SkillPackageWriteOptions(limits)); + + Assert.assertSame(limits, observed[0]); + } + + /** + * noop 业务校验器不能绕过 Codec 内建的 YAML 与包大小安全校验。 + */ + @Test + public void noopValidatorCannotBypassMandatoryYamlAndPackageLimits() { + SkillValidator noopValidator = skill -> { + // 业务层不增加规则,Codec 仍必须独立完成标准安全校验。 + }; + ZipSkillPackageCodec codec = new ZipSkillPackageCodec( + new InMemorySkillContentStore(), noopValidator); + Skill malformedYaml = SkillFactory.create("id", skillMd("skill-a")); + malformedYaml.setSkillContent("---\nname: skill-a\ndescription: [\n---\n# Invalid\n"); + assertExportIssue(codec, malformedYaml, SkillPackageWriteOptions.defaults(), + "INVALID_FRONTMATTER_YAML", SkillPaths.SKILL_FILE); + + Skill oversized = SkillFactory.create("id", skillMd("skill-a") + "x".repeat(300)); + SkillPackageLimits limits = SkillPackageLimits.builder() + .maxTextFileBytes(128) + .build(); + assertExportIssue(codec, oversized, new SkillPackageWriteOptions(limits), + "TEXT_FILE_SIZE_LIMIT", SkillPaths.SKILL_FILE); + } + + /** + * Codec 强制标准校验后仍执行调用方注入的业务校验器。 + */ + @Test + public void preserveAdditionalBusinessValidator() { + SkillValidator businessValidator = skill -> { + throw new SkillValidationException("CUSTOM_POLICY", SkillPaths.SKILL_FILE, + null, null, "Custom Skill policy rejected the package.", null); + }; + ZipSkillPackageCodec codec = new ZipSkillPackageCodec( + new InMemorySkillContentStore(), businessValidator); + + assertExportIssue(codec, SkillFactory.create("id", skillMd("skill-a")), + SkillPackageWriteOptions.defaults(), "CUSTOM_POLICY", SkillPaths.SKILL_FILE); + } + + /** + * SKILL.md 与普通文本资源中的孤立 UTF-16 surrogate 均以结构化错误拒绝。 + */ + @Test + public void rejectMalformedUtf16SurrogateDuringExport() { + String isolatedSurrogate = String.valueOf((char) 0xD800); + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + Skill invalidDocument = SkillFactory.create("id", skillMd("skill-a")); + invalidDocument.setSkillContent(skillMd("skill-a") + isolatedSurrogate); + assertExportIssue(codec, invalidDocument, SkillPackageWriteOptions.defaults(), + "INVALID_UTF8", SkillPaths.SKILL_FILE); + + Skill invalidResource = SkillFactory.create("id", skillMd("skill-a")); + SkillResource resource = new SkillResource(); + resource.setPath("references/invalid.md"); + resource.setKind(SkillResourceKind.REFERENCE); + resource.setMediaType("text/markdown"); + resource.setTextContent(isolatedSurrogate); + resource.setContentHash("0".repeat(64)); + resource.setSize(1); + invalidResource.setResources(List.of(resource)); + assertExportIssue(codec, invalidResource, SkillPackageWriteOptions.defaults(), + "INVALID_UTF8", "references/invalid.md"); + + Skill invalidPath = SkillFactory.create("id", skillMd("skill-a")); + SkillResource pathResource = new SkillResource(); + String malformedPath = "references/" + isolatedSurrogate + ".md"; + pathResource.setPath(malformedPath); + pathResource.setKind(SkillResourceKind.REFERENCE); + pathResource.setMediaType("text/markdown"); + pathResource.setTextContent("valid"); + pathResource.setContentHash(SkillHashes.sha256Hex(utf8("valid"))); + pathResource.setSize(utf8("valid").length); + invalidPath.setResources(List.of(pathResource)); + assertExportIssue(codec, invalidPath, SkillPackageWriteOptions.defaults(), + "INVALID_UTF8_PATH", malformedPath); + } + + /** + * 缺失资源路径在标准导出前返回结构化报告,且不能写出部分 ZIP。 + */ + @Test + public void rejectMissingResourcePathBeforeExportWritesBytes() { + Skill skill = SkillFactory.create("id", skillMd("skill-a")); + SkillResource resource = new SkillResource(); + resource.setKind(SkillResourceKind.REFERENCE); + resource.setMediaType("text/markdown"); + resource.setTextContent("missing path"); + resource.setContentHash(SkillHashes.sha256Hex(utf8("missing path"))); + resource.setSize(utf8("missing path").length); + skill.setResources(List.of(resource)); + + assertExportIssue(new ZipSkillPackageCodec(), skill, + SkillPackageWriteOptions.defaults(), "RESOURCE_PATH_REQUIRED", null); + } + + /** + * 历史下划线名称可导入草稿,但不能标准导出。 + */ + @Test + public void rejectLegacyNameOnStandardExport() { + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + SkillPackage skillPackage = decode(codec, zip(files( + "legacy_skill/SKILL.md", utf8(skillMd("legacy_skill")) + ))).getSkillPackage(); try { - new ZipSkillPackageCodec(store).importZip(new ByteArrayInputStream(zip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/assets/data.bin", "abc", - "skill-a/unknown/a.md", "# Unknown" - )))); - Assert.fail("Import should fail."); - } catch (SkillPackageException expected) { - Assert.assertEquals(0, store.putCount); + codec.encode(skillPackage, new ByteArrayOutputStream(), SkillPackageWriteOptions.defaults()); + Assert.fail("Legacy name should not be exported."); + } catch (SkillPackageException e) { + Assert.assertNotNull(e.getReport()); + Assert.assertTrue(e.getReport().getIssues().stream().anyMatch(issue -> + "NON_CANONICAL_NAME".equals(issue.getCode()) + && issue.getSeverity() + == com.easyagents.skill.validation.SkillValidationSeverity.ERROR)); } } /** - * 多 Skill 包中后续 Skill 失败时,不提前写入前面 Skill 的资产。 + * 目录名与 frontmatter 名称不一致时返回结构化错误。 */ @Test - public void failedMultiSkillImportDoesNotWriteEarlierAssetContent() { - CountingContentStore store = new CountingContentStore(); - + public void rejectRootNameMismatch() { try { - new ZipSkillPackageCodec(store).importZip(new ByteArrayInputStream(zip(files( - "skill-a/SKILL.md", skillMd("Skill A", "Desc A"), - "skill-a/assets/data.bin", "abc", - "skill-b/references/a.md", "# Missing SKILL.md" - )))); - Assert.fail("Import should fail."); - } catch (SkillPackageException expected) { - Assert.assertEquals(0, store.putCount); + decode(new ZipSkillPackageCodec(), zip(files( + "folder-name/SKILL.md", utf8(skillMd("different-name")) + ))); + Assert.fail("Mismatched root should fail."); + } catch (SkillPackageException e) { + Assert.assertNotNull(e.getReport()); + Assert.assertTrue(e.getReport().getIssues().stream().anyMatch(issue -> + "ROOT_NAME_MISMATCH".equals(issue.getCode()))); } } /** - * 拒绝嵌套 frontmatter。 + * 导出先聚合校验错误,不因缺失 SKILL.md 内容触发空指针。 */ - @Test(expected = SkillPackageException.class) - public void rejectNestedFrontmatter() { - importZip(files( - "skill-a/SKILL.md", "---\nname: Skill A\ndescription: Desc A\nconfig:\n mode: strict\n---\n# Skill A\n" - )); + @Test + public void aggregateInvalidSkillBeforePreparingOutput() { + Skill skill = com.easyagents.skill.factory.SkillFactory.create("id", skillMd("skill-a")); + skill.setSkillContent(null); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try { + new ZipSkillPackageCodec().encode( + new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)), output, + SkillPackageWriteOptions.defaults()); + Assert.fail("Invalid Skill should fail."); + } catch (SkillPackageException e) { + Assert.assertNotNull(e.getReport()); + Assert.assertTrue(e.getReport().getIssues().stream().anyMatch(issue -> + "SKILL_CONTENT_REQUIRED".equals(issue.getCode()))); + Assert.assertEquals(0, output.size()); + } } - private static List importZip(Map files) { - return new ZipSkillPackageCodec().importZip(new ByteArrayInputStream(zip(files))); + /** + * 普通文本资源导出也执行单文件限制。 + */ + @Test + public void enforceTextLimitDuringExport() { + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + SkillPackage skillPackage = decode(codec, zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/references/a.md", utf8("x".repeat(300)) + ))).getSkillPackage(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try { + codec.encode(skillPackage, output, new SkillPackageWriteOptions( + SkillPackageLimits.builder().maxTextFileBytes(128).build())); + Assert.fail("Oversized text export should fail."); + } catch (SkillPackageException e) { + Assert.assertTrue(e.getReport().getIssues().stream().anyMatch(issue -> + "TEXT_FILE_SIZE_LIMIT".equals(issue.getCode()))); + Assert.assertEquals(0, output.size()); + } } - private static Map files(String... keyValues) { - Map files = new LinkedHashMap<>(); - for (int i = 0; i < keyValues.length; i += 2) { - files.put(keyValues[i], keyValues[i + 1]); + /** + * 输出压缩包超限返回明确错误码。 + */ + @Test + public void reportCompressedOutputLimit() { + ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + SkillPackage skillPackage = decode(codec, zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")) + ))).getSkillPackage(); + + try { + codec.encode(skillPackage, new ByteArrayOutputStream(), new SkillPackageWriteOptions( + SkillPackageLimits.builder().maxCompressedPackageBytes(64).build())); + Assert.fail("Compressed output limit should fail."); + } catch (SkillPackageException e) { + Assert.assertEquals("COMPRESSED_SIZE_LIMIT", e.getCode()); + } + } + + /** + * 二进制存储内容在写 ZIP 前完成 size/hash 预检。 + */ + @Test + public void preflightBinaryContentBeforeWritingArchive() { + byte[] expected = utf8("abc"); + String hash = SkillHashes.sha256Hex(expected); + SkillResource resource = new SkillResource(); + resource.setPath("assets/data.bin"); + resource.setKind(SkillResourceKind.ASSET); + resource.setMediaType("application/octet-stream"); + resource.setContentRef("sha256:" + hash); + resource.setContentHash(hash); + resource.setSize(expected.length); + Skill skill = com.easyagents.skill.factory.SkillFactory.create("id", skillMd("skill-a")); + skill.setResources(List.of(resource)); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try { + new ZipSkillPackageCodec(new CorruptReadContentStore()).encode( + new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)), output, + SkillPackageWriteOptions.defaults()); + Assert.fail("Corrupt stored content should fail preflight."); + } catch (SkillPackageException e) { + Assert.assertEquals("RESOURCE_HASH_MISMATCH", e.getCode()); + Assert.assertEquals(0, output.size()); + } + } + + private static SkillPackageReadResult decode(ZipSkillPackageCodec codec, byte[] bytes) { + return codec.decode(new ByteArrayInputStream(bytes), SkillPackageReadOptions.defaults()); + } + + private static void assertExportIssue(ZipSkillPackageCodec codec, Skill skill, + SkillPackageWriteOptions options, + String code, String path) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try { + codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)), + output, options); + Assert.fail("Skill export should fail with " + code + "."); + } catch (SkillPackageException e) { + Assert.assertNotNull(e.getReport()); + Assert.assertTrue(e.getReport().getIssues().stream().anyMatch(issue -> + code.equals(issue.getCode()) && Objects.equals(path, issue.getPath()))); + Assert.assertEquals(0, output.size()); + } + } + + private static void assertPackageCode(String code, byte[] bytes, SkillPackageReadOptions options) { + try { + new ZipSkillPackageCodec().decode(new ByteArrayInputStream(bytes), options); + Assert.fail("Package should fail with " + code + "."); + } catch (SkillPackageException e) { + Assert.assertEquals(code, e.getCode()); + } + } + + private static Map files(Object... keyValues) { + Map files = new LinkedHashMap<>(); + for (int index = 0; index < keyValues.length; index += 2) { + files.put((String) keyValues[index], (byte[]) keyValues[index + 1]); } return files; } - private static byte[] zip(Map files, String... directories) { + private static byte[] zip(Map files, String... directories) { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - try (ZipOutputStream zipOutputStream = new ZipOutputStream(bytes)) { + try (ZipOutputStream zipOutput = new ZipOutputStream(bytes, StandardCharsets.UTF_8)) { for (String directory : directories) { - zipOutputStream.putNextEntry(new ZipEntry(directory)); - zipOutputStream.closeEntry(); + zipOutput.putNextEntry(new ZipEntry(directory)); + zipOutput.closeEntry(); } - for (Map.Entry file : files.entrySet()) { - zipOutputStream.putNextEntry(new ZipEntry(file.getKey())); - zipOutputStream.write(file.getValue().getBytes(StandardCharsets.UTF_8)); - zipOutputStream.closeEntry(); + for (Map.Entry file : files.entrySet()) { + zipOutput.putNextEntry(new ZipEntry(file.getKey())); + zipOutput.write(file.getValue()); + zipOutput.closeEntry(); } } return bytes.toByteArray(); @@ -243,8 +1047,242 @@ public class ZipSkillPackageCodecTest { } } - private static String skillMd(String name, String description) { - return "---\nname: " + name + "\ndescription: " + description + "\n---\n# " + name + "\n"; + /** + * 构造仅用于中央目录预扫描测试的 ZIP;本地头无有效 entry,旧 ZipFile 路径无法安全读取。 + * + * @param actualEntries 中央目录实际条目数 + * @param declaredEntries EOCD 声明条目数 + * @return 合成 ZIP 字节 + */ + private static byte[] syntheticCentralDirectoryZip(int actualEntries, int declaredEntries) { + int localHeaderSize = 30; + int nameSize = 2; + int centralEntrySize = 46 + nameSize; + int centralSize = actualEntries * centralEntrySize; + ByteBuffer buffer = ByteBuffer.allocate(localHeaderSize + centralSize + 22) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(0x04034B50); + buffer.position(localHeaderSize); + writeSyntheticCentralDirectoryEntries(buffer, actualEntries, nameSize); + buffer.putInt(0x06054B50); + buffer.putShort((short) 0); + buffer.putShort((short) 0); + buffer.putShort((short) declaredEntries); + buffer.putShort((short) declaredEntries); + buffer.putInt(centralSize); + buffer.putInt(localHeaderSize); + buffer.putShort((short) 0); + return buffer.array(); + } + + /** + * 构造带 ZIP64 EOCD 的中央目录预扫描样本。 + * + * @param actualEntries 中央目录实际条目数 + * @param declaredEntries ZIP64 EOCD 声明条目数 + * @return 合成 ZIP64 字节 + */ + private static byte[] syntheticZip64CentralDirectoryZip( + int actualEntries, int declaredEntries) { + int localHeaderSize = 30; + int nameSize = 2; + int centralEntrySize = 46 + nameSize; + int centralSize = actualEntries * centralEntrySize; + int zip64EndRecordSize = 56; + int zip64LocatorSize = 20; + int endRecordSize = 22; + int zip64EndRecordOffset = localHeaderSize + centralSize; + ByteBuffer buffer = ByteBuffer.allocate(localHeaderSize + centralSize + + zip64EndRecordSize + zip64LocatorSize + endRecordSize) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(0x04034B50); + buffer.position(localHeaderSize); + writeSyntheticCentralDirectoryEntries(buffer, actualEntries, nameSize); + + buffer.putInt(0x06064B50); + buffer.putLong(44L); + buffer.putShort((short) 45); + buffer.putShort((short) 45); + buffer.putInt(0); + buffer.putInt(0); + buffer.putLong(declaredEntries); + buffer.putLong(declaredEntries); + buffer.putLong(centralSize); + buffer.putLong(localHeaderSize); + + buffer.putInt(0x07064B50); + buffer.putInt(0); + buffer.putLong(zip64EndRecordOffset); + buffer.putInt(1); + + buffer.putInt(0x06054B50); + buffer.putShort((short) 0); + buffer.putShort((short) 0); + buffer.putShort((short) 0xFFFF); + buffer.putShort((short) 0xFFFF); + buffer.putInt(0xFFFFFFFF); + buffer.putInt(0xFFFFFFFF); + buffer.putShort((short) 0); + return buffer.array(); + } + + /** + * 写入只包含最小必要字段的中央目录条目。 + * + * @param buffer 目标小端缓冲区 + * @param entryCount 条目数 + * @param nameSize 文件名长度 + */ + private static void writeSyntheticCentralDirectoryEntries( + ByteBuffer buffer, int entryCount, int nameSize) { + for (int index = 0; index < entryCount; index++) { + buffer.putInt(0x02014B50); + buffer.putShort((short) 20); + buffer.putShort((short) 20); + buffer.position(buffer.position() + 20); + buffer.putShort((short) nameSize); + buffer.putShort((short) 0); + buffer.putShort((short) 0); + buffer.position(buffer.position() + 12); + buffer.put((byte) 'e'); + buffer.put((byte) ('0' + index)); + } + } + + /** + * 删除中央目录最后一个字节并同步 EOCD 的目录大小,模拟结构性截断。 + * + * @param source 无注释的传统 ZIP 样本 + * @return 中央目录被截断的 ZIP 字节 + */ + private static byte[] truncateLastCentralDirectoryByte(byte[] source) { + int endRecordSize = 22; + int endRecordOffset = source.length - endRecordSize; + int removedOffset = endRecordOffset - 1; + ByteBuffer sourceBuffer = ByteBuffer.wrap(source).order(ByteOrder.LITTLE_ENDIAN); + int centralSize = sourceBuffer.getInt(endRecordOffset + 12); + byte[] truncated = new byte[source.length - 1]; + System.arraycopy(source, 0, truncated, 0, removedOffset); + System.arraycopy(source, removedOffset + 1, truncated, removedOffset, + source.length - removedOffset - 1); + ByteBuffer.wrap(truncated).order(ByteOrder.LITTLE_ENDIAN) + .putInt(endRecordOffset - 1 + 12, centralSize - 1); + return truncated; + } + + private static byte[] symlinkZip() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipArchiveOutputStream output = new ZipArchiveOutputStream(bytes)) { + ZipArchiveEntry skillEntry = new ZipArchiveEntry("skill-a/SKILL.md"); + skillEntry.setUnixMode(UnixStat.FILE_FLAG | 0644); + output.putArchiveEntry(skillEntry); + output.write(utf8(skillMd("skill-a"))); + output.closeArchiveEntry(); + + ZipArchiveEntry symlink = new ZipArchiveEntry("skill-a/assets/link"); + symlink.setUnixMode(UnixStat.LINK_FLAG | 0777); + output.putArchiveEntry(symlink); + output.write(utf8("target")); + output.closeArchiveEntry(); + output.finish(); + } + return bytes.toByteArray(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static byte[] invalidUtf8EntryNameZip() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipArchiveOutputStream output = new ZipArchiveOutputStream(bytes)) { + output.setEncoding(StandardCharsets.ISO_8859_1.name()); + output.setUseLanguageEncodingFlag(false); + output.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.NEVER); + + ZipArchiveEntry skillEntry = new ZipArchiveEntry("skill-a/SKILL.md"); + output.putArchiveEntry(skillEntry); + output.write(utf8(skillMd("skill-a"))); + output.closeArchiveEntry(); + + ZipArchiveEntry invalidName = new ZipArchiveEntry("skill-a/references/\u00ff.md"); + output.putArchiveEntry(invalidName); + output.write(utf8("invalid name")); + output.closeArchiveEntry(); + output.finish(); + } + return bytes.toByteArray(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static byte[] tamperFirstCentralDirectoryCrc(byte[] source) { + byte[] bytes = Arrays.copyOf(source, source.length); + for (int index = 0; index <= bytes.length - 20; index++) { + if (bytes[index] == 0x50 && bytes[index + 1] == 0x4B + && bytes[index + 2] == 0x01 && bytes[index + 3] == 0x02) { + bytes[index + 16] ^= 0x01; + return bytes; + } + } + throw new IllegalStateException("ZIP central directory was not found."); + } + + private static boolean allEntriesHaveStableTimestamp(byte[] zip) { + List timestamps = new ArrayList<>(); + try (ZipInputStream input = new ZipInputStream(new ByteArrayInputStream(zip), StandardCharsets.UTF_8)) { + ZipEntry entry; + while ((entry = input.getNextEntry()) != null) { + timestamps.add(entry.getTime()); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + return !timestamps.isEmpty() && timestamps.stream().distinct().count() == 1; + } + + private static int zipEntryMethod(byte[] zip, String path) { + try (ZipInputStream input = new ZipInputStream( + new ByteArrayInputStream(zip), StandardCharsets.UTF_8)) { + ZipEntry entry; + while ((entry = input.getNextEntry()) != null) { + if (path.equals(entry.getName())) { + return entry.getMethod(); + } + } + throw new IllegalStateException("ZIP entry was not found: " + path); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static boolean zipContains(byte[] zip, String value) { + try (ZipInputStream input = new ZipInputStream(new ByteArrayInputStream(zip), StandardCharsets.UTF_8)) { + ZipEntry entry; + while ((entry = input.getNextEntry()) != null) { + if (entry.getName().contains(value) + || new String(input.readAllBytes(), StandardCharsets.UTF_8).contains(value)) { + return true; + } + } + return false; + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static byte[] utf8(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static String skillMd(String name) { + return "---\nname: " + name + "\ndescription: Use this Skill for package tests\n---\n# " + name + "\n"; + } + + private static String invalidDescriptionSkillMd(String name) { + return "---\nname: " + name + "\ndescription: ''\n---\n# " + name + "\n"; } private static final class CountingContentStore implements SkillContentStore { @@ -254,7 +1292,7 @@ public class ZipSkillPackageCodecTest { @Override public String put(byte[] bytes) { putCount++; - return "sha256:" + com.easyagents.skill.util.SkillHashes.sha256Hex(bytes); + return SkillHashes.sha256Ref(bytes); } @Override @@ -272,4 +1310,94 @@ public class ZipSkillPackageCodecTest { return false; } } + + private static final class TrackingContentStore implements SkillContentStore { + + private final InMemorySkillContentStore delegate = new InMemorySkillContentStore(); + private final int failOnCommit; + private int stageCalls; + private int stagedCount; + private int committedCount; + private int commitAttempts; + private int rollbackCalls; + + private TrackingContentStore(int failOnCommit) { + this.failOnCommit = failOnCommit; + } + + @Override + public String put(byte[] bytes) { + return delegate.put(bytes); + } + + @Override + public SkillContentStage stage(InputStream inputStream, long maxBytes) { + stageCalls++; + stagedCount++; + return delegate.stage(inputStream, maxBytes); + } + + @Override + public String commit(SkillContentStage stage) { + commitAttempts++; + if (failOnCommit > 0 && commitAttempts == failOnCommit) { + throw new IllegalStateException("simulated commit failure"); + } + String contentRef = delegate.commit(stage); + stagedCount--; + committedCount++; + return contentRef; + } + + @Override + public void rollback(SkillContentStage stage) { + rollbackCalls++; + delegate.rollback(stage); + stagedCount--; + } + + @Override + public void release(String contentRef) { + delegate.release(contentRef); + committedCount--; + } + + @Override + public InputStream open(String contentRef) { + return delegate.open(contentRef); + } + + @Override + public byte[] readAllBytes(String contentRef) { + return delegate.readAllBytes(contentRef); + } + + @Override + public boolean exists(String contentRef) { + return delegate.exists(contentRef); + } + } + + private static final class CorruptReadContentStore implements SkillContentStore { + + @Override + public String put(byte[] bytes) { + return SkillHashes.sha256Ref(bytes); + } + + @Override + public InputStream open(String contentRef) { + return new ByteArrayInputStream(utf8("xyz")); + } + + @Override + public byte[] readAllBytes(String contentRef) { + return utf8("xyz"); + } + + @Override + public boolean exists(String contentRef) { + return true; + } + } } diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillMetadataTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillMetadataTest.java new file mode 100644 index 0000000..d2d232c --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillMetadataTest.java @@ -0,0 +1,50 @@ +package com.easyagents.skill.model; + +import com.easyagents.skill.exception.SkillValidationException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * SkillMetadata 防御性复制与键约束测试。 + */ +public class SkillMetadataTest { + + /** + * 嵌套 Map/List 在读写边界上执行防御性复制。 + */ + @Test + @SuppressWarnings("unchecked") + public void deeplyCopiesNestedValues() { + LinkedHashMap nested = new LinkedHashMap<>(); + nested.put("items", new java.util.ArrayList<>(List.of("a"))); + SkillMetadata metadata = new SkillMetadata(Map.of("nested", nested)); + + Map read = metadata.getValues(); + ((Map) read.get("nested")).put("changed", true); + + Assert.assertNull(((Map) metadata.get("nested")).get("changed")); + } + + /** + * 空白键写入会显式失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectBlankKey() { + new SkillMetadata().put(" ", "value"); + } + + /** + * 嵌套非字符串键会显式失败且不会静默转换。 + */ + @Test(expected = SkillValidationException.class) + @SuppressWarnings({"rawtypes", "unchecked"}) + public void rejectNestedNonStringKey() { + Map nested = new LinkedHashMap(); + nested.put(1, "number"); + new SkillMetadata(Map.of("nested", nested)); + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillPackageLimitsTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillPackageLimitsTest.java new file mode 100644 index 0000000..c57cab0 --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/model/SkillPackageLimitsTest.java @@ -0,0 +1,30 @@ +package com.easyagents.skill.model; + +import com.easyagents.skill.exception.SkillValidationException; +import org.junit.Test; + +/** + * SkillPackageLimits 硬上限测试。 + */ +public class SkillPackageLimitsTest { + + /** + * 调用方不能关闭或突破底层 entry 硬上限。 + */ + @Test(expected = SkillValidationException.class) + public void rejectEntryCountAboveHardLimit() { + SkillPackageLimits.builder().maxEntryCount(10_001).build(); + } + + /** + * 单文件限制不能大于总未压缩限制。 + */ + @Test(expected = SkillValidationException.class) + public void rejectSingleFileLimitAboveTotalLimit() { + SkillPackageLimits.builder() + .maxTextFileBytes(200) + .maxBinaryFileBytes(200) + .maxTotalUncompressedBytes(100) + .build(); + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java index fa3a550..b2371a9 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java @@ -3,6 +3,7 @@ package com.easyagents.skill.repository.memory; import com.easyagents.skill.model.Skill; import com.easyagents.skill.model.SkillDescriptor; import com.easyagents.skill.model.SkillReference; +import com.easyagents.skill.util.SkillResources; import org.junit.Assert; import org.junit.Test; @@ -66,6 +67,40 @@ public class InMemorySkillRepositoryTest { Assert.assertEquals("# A", reloaded.getReferences().get(0).getContent()); } + /** + * 仓储复制旧资源对象时应先迁移正式资源,不能因空 canonical 列表丢失 reference。 + */ + @Test + public void repositoryCopyMigratesLegacyOnlyResources() { + InMemorySkillRepository repository = new InMemorySkillRepository(); + Skill legacy = skill(); + Assert.assertFalse(legacy.isResourcesInitialized()); + + repository.save(legacy); + Skill loaded = repository.get("skill-a").orElseThrow(); + + Assert.assertTrue(loaded.isResourcesInitialized()); + Assert.assertEquals(1, SkillResources.canonicalResources(loaded).size()); + Assert.assertEquals("references/a.md", loaded.getResources().get(0).getPath()); + } + + /** + * 显式清空正式资源列表后,仓储往返不得从旧兼容视图恢复已删除资源。 + */ + @Test + public void repositoryCopyPreservesExplicitlyEmptyCanonicalResources() { + InMemorySkillRepository repository = new InMemorySkillRepository(); + Skill skill = skill(); + Assert.assertEquals(1, skill.getResources().size()); + skill.getResources().clear(); + + repository.save(skill); + Skill loaded = repository.get("skill-a").orElseThrow(); + + Assert.assertTrue(loaded.isResourcesInitialized()); + Assert.assertTrue(SkillResources.canonicalResources(loaded).isEmpty()); + } + private static Skill skill() { Skill skill = new Skill(); skill.setId("skill-a"); diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java new file mode 100644 index 0000000..43b4f38 --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java @@ -0,0 +1,275 @@ +package com.easyagents.skill.store.file; + +import com.easyagents.skill.codec.ZipSkillPackageCodec; +import com.easyagents.skill.exception.SkillException; +import com.easyagents.skill.store.SkillContentStage; +import com.easyagents.skill.store.SkillContentStore; +import com.easyagents.skill.store.memory.InMemorySkillContentStore; +import com.easyagents.skill.util.SkillHashes; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +/** + * {@link TemporaryFileSkillContentStore} 流式存储与生命周期测试。 + */ +public class TemporaryFileSkillContentStoreTest { + + /** + * 暂存、提交、回滚、去重与引用计数均由磁盘文件驱动。 + * + * @throws Exception 文件操作失败时抛出 + */ + @Test + public void stageCommitRollbackAndReferenceCountingAreFileBacked() throws Exception { + Path parent = Files.createTempDirectory("skill-content-store-test-"); + TemporaryFileSkillContentStore store = new TemporaryFileSkillContentStore(parent); + Path storageDirectory = store.storageDirectory(); + byte[] content = "file-backed-content".getBytes(StandardCharsets.UTF_8); + try { + SkillContentStage rolledBack = store.stage( + new ByteArrayInputStream(content), content.length); + Assert.assertFalse(store.exists(rolledBack.getContentRef())); + Assert.assertEquals(1, countRegularFiles(storageDirectory.resolve("staged"))); + + store.rollback(rolledBack); + Assert.assertEquals(0, countRegularFiles(storageDirectory.resolve("staged"))); + + SkillContentStage firstStage = store.stage( + new ByteArrayInputStream(content), content.length); + SkillContentStage secondStage = store.stage( + new ByteArrayInputStream(content), content.length); + String firstRef = store.commit(firstStage); + String secondRef = store.commit(secondStage); + + Assert.assertEquals(firstRef, secondRef); + Assert.assertTrue(store.exists(firstRef)); + Assert.assertEquals(0, countRegularFiles(storageDirectory.resolve("staged"))); + Assert.assertEquals(1, countRegularFiles(storageDirectory.resolve("content"))); + + store.release(firstRef); + Assert.assertTrue(store.exists(secondRef)); + store.retain(secondRef); + store.release(secondRef); + Assert.assertTrue(store.exists(secondRef)); + store.release(secondRef); + + Assert.assertFalse(store.exists(secondRef)); + Assert.assertEquals(0, countRegularFiles(storageDirectory.resolve("content"))); + } finally { + store.close(); + Assert.assertFalse(Files.exists(storageDirectory)); + Assert.assertTrue(Files.exists(parent)); + Files.deleteIfExists(parent); + } + } + + /** + * 已打开的流持有读取租约,最后一个正式引用释放后等待流关闭再删除文件。 + * + * @throws Exception 文件读取失败时抛出 + */ + @Test + public void openStreamDefersPhysicalDeletionUntilReaderCloses() throws Exception { + Path parent = Files.createTempDirectory("skill-content-reader-test-"); + try (TemporaryFileSkillContentStore store = new TemporaryFileSkillContentStore(parent)) { + Path contentDirectory = store.storageDirectory().resolve("content"); + byte[] content = "leased-content".getBytes(StandardCharsets.UTF_8); + String contentRef = store.put(content); + InputStream input = store.open(contentRef); + + store.release(contentRef); + Assert.assertFalse(store.exists(contentRef)); + Assert.assertEquals(1, countRegularFiles(contentDirectory)); + Assert.assertArrayEquals(content, input.readAllBytes()); + + input.close(); + Assert.assertEquals(0, countRegularFiles(contentDirectory)); + + String pendingRef = store.put(content); + InputStream pendingReader = store.open(pendingRef); + store.release(pendingRef); + String revivedRef = store.put(content); + Assert.assertEquals(pendingRef, revivedRef); + + pendingReader.close(); + Assert.assertTrue(store.exists(revivedRef)); + Assert.assertEquals(1, countRegularFiles(contentDirectory)); + store.release(revivedRef); + Assert.assertEquals(0, countRegularFiles(contentDirectory)); + } finally { + Files.deleteIfExists(parent); + } + } + + /** + * 大内容按流写入并受字节上限约束,存储实现及内部状态不持有 byte[] 字段。 + * + * @throws Exception 文件操作失败时抛出 + */ + @Test + public void largeContentIsStreamedWithoutByteArrayState() throws Exception { + assertNoByteArrayFields(TemporaryFileSkillContentStore.class); + long contentSize = 2L * 1024 * 1024 + 37; + Path parent = Files.createTempDirectory("skill-content-stream-test-"); + try (TemporaryFileSkillContentStore store = new TemporaryFileSkillContentStore(parent)) { + SkillContentStage stage = store.stage(new RepeatingInputStream(contentSize), contentSize); + Assert.assertEquals(contentSize, stage.getSize()); + String contentRef = store.commit(stage); + + MessageDigest digest = SkillHashes.newSha256Digest(); + long copied = 0; + try (InputStream input = store.open(contentRef)) { + byte[] buffer = new byte[8192]; + int length; + while ((length = input.read(buffer)) >= 0) { + if (length == 0) { + continue; + } + digest.update(buffer, 0, length); + copied += length; + } + } + Assert.assertEquals(contentSize, copied); + Assert.assertEquals(stage.getContentHash(), SkillHashes.toHex(digest.digest())); + + assertLimitExceeded(store, contentSize - 1); + Assert.assertEquals(0, + countRegularFiles(store.storageDirectory().resolve("staged"))); + store.release(contentRef); + } finally { + Files.deleteIfExists(parent); + } + } + + /** + * 无参 ZIP Codec 使用自有临时文件 Store,注入 Store 仍由调用方管理。 + * + * @throws Exception 反射或文件操作失败时抛出 + */ + @Test + public void defaultCodecOwnsTemporaryStoreWhileInjectedStoreRemainsExternal() + throws Exception { + Field contentStoreField = ZipSkillPackageCodec.class.getDeclaredField("contentStore"); + contentStoreField.setAccessible(true); + + ZipSkillPackageCodec defaultCodec = new ZipSkillPackageCodec(); + SkillContentStore defaultStore = (SkillContentStore) contentStoreField.get(defaultCodec); + Assert.assertTrue(defaultStore instanceof TemporaryFileSkillContentStore); + Assert.assertFalse(defaultStore instanceof InMemorySkillContentStore); + Path defaultDirectory = ((TemporaryFileSkillContentStore) defaultStore).storageDirectory(); + defaultCodec.close(); + Assert.assertFalse(Files.exists(defaultDirectory)); + + Path parent = Files.createTempDirectory("skill-codec-external-store-test-"); + TemporaryFileSkillContentStore injectedStore = new TemporaryFileSkillContentStore(parent); + try { + ZipSkillPackageCodec injectedCodec = new ZipSkillPackageCodec(injectedStore); + injectedCodec.close(); + String contentRef = injectedStore.put("still-open".getBytes(StandardCharsets.UTF_8)); + Assert.assertTrue(injectedStore.exists(contentRef)); + injectedStore.release(contentRef); + } finally { + injectedStore.close(); + Files.deleteIfExists(parent); + } + } + + /** + * 显式关闭后清理暂存文件并拒绝新的写入。 + * + * @throws Exception 文件操作失败时抛出 + */ + @Test + public void closeCleansStagesAndRejectsFurtherOperations() throws Exception { + Path parent = Files.createTempDirectory("skill-content-close-test-"); + TemporaryFileSkillContentStore store = new TemporaryFileSkillContentStore(parent); + Path storageDirectory = store.storageDirectory(); + store.stage(new ByteArrayInputStream(new byte[]{1, 2, 3}), 3); + + store.close(); + store.close(); + + Assert.assertFalse(Files.exists(storageDirectory)); + try { + store.stage(new ByteArrayInputStream(new byte[0]), 0); + Assert.fail("Closed store should reject new stages."); + } catch (SkillException expected) { + Assert.assertTrue(expected.getMessage().contains("closed")); + } finally { + Files.deleteIfExists(parent); + } + } + + private static void assertLimitExceeded(TemporaryFileSkillContentStore store, long maxBytes) { + try { + store.stage(new RepeatingInputStream(maxBytes + 1), maxBytes); + Assert.fail("Content above the limit should be rejected."); + } catch (SkillException expected) { + Assert.assertTrue(expected.getMessage().contains("exceeds")); + } + } + + private static void assertNoByteArrayFields(Class rootType) { + List> types = new ArrayList<>(); + types.add(rootType); + for (int index = 0; index < types.size(); index++) { + Class type = types.get(index); + for (Field field : type.getDeclaredFields()) { + Assert.assertNotEquals(type.getName() + " must not retain byte[] field " + + field.getName(), byte[].class, field.getType()); + } + for (Class nestedType : type.getDeclaredClasses()) { + types.add(nestedType); + } + } + } + + private static long countRegularFiles(Path directory) throws IOException { + try (Stream paths = Files.list(directory)) { + return paths.filter(Files::isRegularFile).count(); + } + } + + private static final class RepeatingInputStream extends InputStream { + + private final long length; + private long position; + + private RepeatingInputStream(long length) { + this.length = length; + } + + @Override + public int read() { + if (position >= length) { + return -1; + } + return (int) (position++ & 0xFF); + } + + @Override + public int read(byte[] bytes, int offset, int requestedLength) { + if (position >= length) { + return -1; + } + int actualLength = (int) Math.min(requestedLength, length - position); + for (int index = 0; index < actualLength; index++) { + bytes[offset + index] = (byte) ((position + index) & 0xFF); + } + position += actualLength; + return actualLength; + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java index 1050cbe..e5f14d3 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java @@ -1,5 +1,6 @@ package com.easyagents.skill.store.memory; +import com.easyagents.skill.store.SkillContentStage; import org.junit.Assert; import org.junit.Test; @@ -43,4 +44,40 @@ public class InMemorySkillContentStoreTest { Assert.assertArrayEquals(bytes, inputStream.readAllBytes()); } } + + /** + * 暂存内容在 commit 前不可见,rollback 后不残留。 + */ + @Test + public void stageCommitAndRollback() { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); + + SkillContentStage rolledBack = store.stage(new java.io.ByteArrayInputStream(bytes), bytes.length); + Assert.assertFalse(store.exists(rolledBack.getContentRef())); + store.rollback(rolledBack); + Assert.assertFalse(store.exists(rolledBack.getContentRef())); + + SkillContentStage committed = store.stage(new java.io.ByteArrayInputStream(bytes), bytes.length); + String contentRef = store.commit(committed); + Assert.assertTrue(store.exists(contentRef)); + Assert.assertArrayEquals(bytes, store.readAllBytes(contentRef)); + } + + /** + * 相同内容按引用计数释放,归零后删除。 + */ + @Test + public void releaseDeletesOnlyAfterReferenceCountReachesZero() { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); + + String first = store.put(bytes); + String second = store.put(bytes); + store.release(first); + + Assert.assertTrue(store.exists(second)); + store.release(second); + Assert.assertFalse(store.exists(second)); + } } diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillFrontmatterTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillFrontmatterTest.java new file mode 100644 index 0000000..764b29e --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillFrontmatterTest.java @@ -0,0 +1,199 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.model.SkillDocument; +import com.easyagents.skill.model.SkillPackageLimits; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +/** + * SkillFrontmatter 安全解析与 round-trip 测试。 + */ +public class SkillFrontmatterTest { + + /** + * 嵌套结构、未知字段和插入顺序在序列化后保持语义。 + */ + @Test + public void nestedMetadataRoundTrips() { + String content = "---\nname: nested-skill\ndescription: Handles nested values\n" + + "enabled: true\ncount: 3\nnullable: null\n" + + "metadata:\n owner: team-a\n tags:\n - one\n - two\n" + + "---\n# Nested\n"; + + SkillDocument first = SkillFrontmatter.parseDocument(content); + String serialized = SkillFrontmatter.serialize(first.getFrontmatter().getValues(), first.getMarkdownBody()); + SkillDocument second = SkillFrontmatter.parseDocument(serialized); + + Assert.assertEquals(first.getFrontmatter().getValues(), second.getFrontmatter().getValues()); + Assert.assertEquals(first.getMarkdownBody(), second.getMarkdownBody()); + Assert.assertEquals(List.of("name", "description", "enabled", "count", "nullable", "metadata"), + second.getFrontmatter().getValues().keySet().stream().toList()); + } + + /** + * 未编辑文档原样返回,结构化编辑后重新生成合法文档。 + */ + @Test + public void preserveRawUntilEdited() { + String content = "---\r\nname: raw-skill\r\ndescription: Keep source\r\n---\r\n# Raw\r\n"; + SkillDocument document = SkillFrontmatter.parseDocument(content); + + Assert.assertEquals(content, document.render()); + + document.putFrontmatter("metadata", Map.of("level", 2)); + Assert.assertNotEquals(content, document.render()); + SkillDocument parsed = SkillFrontmatter.parseDocument(document.render()); + Assert.assertEquals("# Raw\r\n", parsed.getMarkdownBody()); + Assert.assertTrue(parsed.getFrontmatter().get("metadata") instanceof Map); + } + + /** + * 顶层字段保留完整 SKILL.md 中的一基源码位置,结构化修改后移除失效位置。 + */ + @Test + public void retainTopLevelSourceLocationsUntilFieldEdited() { + String content = "---\nname: located-skill\ndescription: Keep positions\n" + + "metadata:\n owner: team-a\n---\n# Located\n"; + + SkillDocument document = SkillFrontmatter.parseDocument(content); + + Assert.assertEquals(2, document.getFrontmatterLocation("name").getLine()); + Assert.assertEquals(1, document.getFrontmatterLocation("name").getColumn()); + Assert.assertEquals(3, document.getFrontmatterLocation("description").getLine()); + Assert.assertEquals(1, document.getFrontmatterLocation("description").getColumn()); + Assert.assertEquals(4, document.getFrontmatterLocation("metadata").getLine()); + Assert.assertEquals(1, document.getFrontmatterLocation("metadata").getColumn()); + Assert.assertTrue(document.getDiagnostics().isEmpty()); + + document.putFrontmatter("description", "Updated description"); + + Assert.assertNull(document.getFrontmatterLocation("description")); + Assert.assertNotNull(document.getFrontmatterLocation("name")); + } + + /** + * 重复 key 被安全构造器拒绝。 + */ + @Test + public void rejectDuplicateKeys() { + assertCode("INVALID_FRONTMATTER_YAML", + "---\nname: duplicate-skill\nname: other\ndescription: Duplicate\n---\n"); + } + + /** + * 自定义危险 tag 被拒绝。 + */ + @Test + public void rejectCustomTag() { + assertCode("INVALID_FRONTMATTER_YAML", + "---\nname: tagged-skill\ndescription: Tagged\nvalue: !java/object test\n---\n"); + } + + /** + * yaml.org 前缀下不在白名单中的 core tag 也被拒绝。 + */ + @Test + public void rejectDisallowedYamlOrgTag() { + assertCode("UNSUPPORTED_FRONTMATTER_TYPE", + "---\nname: binary-skill\ndescription: Binary tag\nvalue: !!binary YWJj\n---\n"); + } + + /** + * 未知 yaml.org 全局 tag 被 TagInspector 拒绝。 + */ + @Test + public void rejectUnknownYamlOrgTag() { + assertCode("INVALID_FRONTMATTER_YAML", + "---\nname: unknown-skill\ndescription: Unknown tag\nvalue: !!unknown test\n---\n"); + } + + /** + * 非 Map 顶层 YAML 被拒绝。 + */ + @Test + public void rejectNonMapRoot() { + assertCode("FRONTMATTER_ROOT_NOT_MAP", "---\n- one\n- two\n---\n"); + } + + /** + * 嵌套 Map 的非字符串 key 被拒绝且不做静默字符串化。 + */ + @Test + public void rejectNestedNonStringKey() { + assertCode("INVALID_FRONTMATTER_KEY", + "---\nname: key-skill\ndescription: Invalid key\nmetadata:\n 1: value\n---\n"); + } + + /** + * 可配置 frontmatter 字节上限生效。 + */ + @Test + public void rejectOversizedFrontmatter() { + SkillPackageLimits limits = SkillPackageLimits.builder().maxFrontmatterBytes(64).build(); + try { + SkillFrontmatter.parseDocument("---\nname: large-skill\ndescription: " + "x".repeat(80) + + "\n---\n", limits); + Assert.fail("Oversized frontmatter should fail."); + } catch (SkillValidationException e) { + Assert.assertEquals("FRONTMATTER_TOO_LARGE", e.getCode()); + } + } + + /** + * YAML collection alias 上限生效。 + */ + @Test + public void rejectAliasLimit() { + String content = "---\nname: alias-skill\ndescription: Alias limit\n" + + "base: &base\n - one\naliases:\n - *base\n - *base\n---\n"; + assertCode("INVALID_FRONTMATTER_YAML", content, + SkillPackageLimits.builder().maxYamlAliases(1).build()); + } + + /** + * YAML 嵌套深度上限生效。 + */ + @Test + public void rejectDepthLimit() { + String content = "---\nname: depth-skill\ndescription: Depth limit\n" + + "metadata:\n one:\n two:\n three: value\n---\n"; + assertCode("INVALID_FRONTMATTER_YAML", content, + SkillPackageLimits.builder().maxYamlDepth(3).build()); + } + + /** + * YAML code point 上限生效。 + */ + @Test + public void rejectCodePointLimit() { + String content = "---\nname: codepoint-skill\ndescription: " + "x".repeat(100) + "\n---\n"; + assertCode("INVALID_FRONTMATTER_YAML", content, + SkillPackageLimits.builder().maxYamlCodePoints(64).build()); + } + + /** + * 未闭合 frontmatter 返回稳定错误码。 + */ + @Test + public void rejectUnclosedFrontmatter() { + assertCode("FRONTMATTER_NOT_CLOSED", + "---\nname: unclosed-skill\ndescription: Unclosed\n"); + } + + private static void assertCode(String code, String content) { + assertCode(code, content, SkillPackageLimits.defaults()); + } + + private static void assertCode(String code, String content, SkillPackageLimits limits) { + try { + SkillFrontmatter.parseDocument(content, limits); + Assert.fail("Invalid frontmatter should fail."); + } catch (SkillValidationException e) { + Assert.assertEquals(code, e.getCode()); + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java new file mode 100644 index 0000000..f2039d7 --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java @@ -0,0 +1,35 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.exception.SkillValidationException; +import org.junit.Assert; +import org.junit.Test; + +/** + * SkillPaths 安全规范化测试。 + */ +public class SkillPathsTest { + + /** + * 点路径段不能被静默剥离。 + */ + @Test(expected = SkillValidationException.class) + public void rejectLeadingDotSegment() { + SkillPaths.normalize("./references/a.md"); + } + + /** + * 双向文本控制字符被拒绝。 + */ + @Test(expected = SkillValidationException.class) + public void rejectBidiFormatCharacter() { + SkillPaths.normalize("references/a\u202Etxt.md"); + } + + /** + * Unicode 路径统一为 NFC。 + */ + @Test + public void normalizeUnicodeToNfc() { + Assert.assertEquals("references/\u00e9.md", SkillPaths.normalize("references/e\u0301.md")); + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java new file mode 100644 index 0000000..555e83d --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java @@ -0,0 +1,22 @@ +package com.easyagents.skill.util; + +import com.easyagents.skill.model.SkillResourceKind; +import org.junit.Assert; +import org.junit.Test; + +/** + * {@link SkillResources} 规范存储表示测试。 + */ +public class SkillResourcesTest { + + /** + * HTML 的短扩展名与长扩展名都应作为严格 UTF-8 文本处理。 + */ + @Test + public void recognizeBothHtmlExtensionsAsText() { + Assert.assertTrue(SkillResources.isText( + "references/page.htm", SkillResourceKind.REFERENCE, "application/octet-stream")); + Assert.assertTrue(SkillResources.isText( + "references/page.html", SkillResourceKind.REFERENCE, "application/octet-stream")); + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/validation/SkillValidationReportTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/SkillValidationReportTest.java new file mode 100644 index 0000000..6e025dc --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/SkillValidationReportTest.java @@ -0,0 +1,37 @@ +package com.easyagents.skill.validation; + +import com.easyagents.skill.exception.SkillValidationException; +import org.junit.Assert; +import org.junit.Test; + +/** + * {@link SkillValidationReport} 命令式异常契约测试。 + */ +public class SkillValidationReportTest { + + /** + * 验证异常 getter 暴露首个错误定位,同时保留包含全部问题的报告。 + */ + @Test + public void throwIfInvalidKeepsFirstErrorLocationAndCompleteReport() { + SkillValidationReport report = new SkillValidationReport() + .add(new SkillValidationIssue("DRAFT_WARNING", SkillValidationSeverity.WARNING, + "draft/SKILL.md", 1, 1, "草稿警告", null)) + .add(new SkillValidationIssue("FIRST_ERROR", SkillValidationSeverity.ERROR, + "demo/SKILL.md", 4, 7, "首个错误", "修复首个错误")) + .add(new SkillValidationIssue("SECOND_ERROR", SkillValidationSeverity.ERROR, + "demo/scripts/run.sh", 2, 3, "第二个错误", null)); + + try { + report.throwIfInvalid(); + Assert.fail("应抛出 SkillValidationException"); + } catch (SkillValidationException exception) { + Assert.assertEquals("FIRST_ERROR", exception.getCode()); + Assert.assertEquals("demo/SKILL.md", exception.getPath()); + Assert.assertEquals(Integer.valueOf(4), exception.getLine()); + Assert.assertEquals(Integer.valueOf(7), exception.getColumn()); + Assert.assertSame(report, exception.getReport()); + Assert.assertEquals(3, exception.getReport().getIssues().size()); + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java index fdaab0a..7af188f 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java @@ -2,8 +2,15 @@ package com.easyagents.skill.validation.defaults; import com.easyagents.skill.exception.SkillValidationException; import com.easyagents.skill.factory.SkillFactory; -import com.easyagents.skill.model.*; +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillResource; +import com.easyagents.skill.model.SkillResourceKind; +import com.easyagents.skill.model.SkillPackageLimits; import com.easyagents.skill.util.SkillHashes; +import com.easyagents.skill.validation.SkillValidationReport; +import com.easyagents.skill.validation.SkillValidationMode; +import com.easyagents.skill.validation.SkillValidationSeverity; +import org.junit.Assert; import org.junit.Test; import java.nio.charset.StandardCharsets; @@ -16,178 +23,385 @@ public class DefaultSkillValidatorTest { private final DefaultSkillValidator validator = new DefaultSkillValidator(); /** - * 缺失 SKILL.md 内容时失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectMissingSkillContent() { - Skill skill = validSkill(); - skill.setSkillContent(null); - - validator.validate(skill); - } - - /** - * 缺失名称时失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectMissingName() { - Skill skill = validSkill(); - skill.setName(""); - - validator.validate(skill); - } - - /** - * 缺失描述时失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectMissingDescription() { - Skill skill = validSkill(); - skill.setDescription(""); - - validator.validate(skill); - } - - /** - * 字段名称必须与 SKILL.md frontmatter 一致。 - */ - @Test(expected = SkillValidationException.class) - public void rejectNameMismatchWithFrontmatter() { - Skill skill = validSkill(); - skill.setName("Changed"); - - validator.validate(skill); - } - - /** - * 元数据必须与 SKILL.md frontmatter 一致。 - */ - @Test(expected = SkillValidationException.class) - public void rejectMetadataMismatchWithFrontmatter() { - Skill skill = validSkill(); - skill.getMetadata().put("extra", "x"); - - validator.validate(skill); - } - - /** - * 拒绝嵌套 frontmatter。 - */ - @Test(expected = SkillValidationException.class) - public void rejectNestedFrontmatter() { - Skill skill = validSkill(); - skill.setSkillContent("---\nname: Skill A\ndescription: Desc A\nconfig:\n mode: strict\n---\n# Skill A\n"); - - validator.validate(skill); - } - - /** - * 绝对路径失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectAbsolutePath() { - Skill skill = validSkill(); - SkillReference reference = reference("/references/a.md", "# A"); - skill.getReferences().add(reference); - - validator.validate(skill); - } - - /** - * ../ 路径失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectParentPath() { - Skill skill = validSkill(); - skill.getReferences().add(reference("references/../a.md", "# A")); - - validator.validate(skill); - } - - /** - * 重复路径失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectDuplicatePath() { - Skill skill = validSkill(); - skill.getReferences().add(reference("references/a.md", "# A")); - skill.getReferences().add(reference("references/a.md", "# B")); - - validator.validate(skill); - } - - /** - * 不因为文件大小较大而失败。 - */ - @Test - public void allowLargeRecordedSize() { - Skill skill = validSkill(); - SkillReference reference = reference("references/a.md", "# A"); - reference.setSize(Long.MAX_VALUE); - skill.getReferences().add(reference); - - validator.validate(skill); - } - - /** - * 允许空内容文件,只校验 hash 和 size 记录。 - */ - @Test - public void allowEmptyReferenceAndScriptContent() { - Skill skill = validSkill(); - skill.getReferences().add(reference("references/empty.md", "")); - skill.getScripts().add(script("scripts/empty.sh", "")); - - validator.validate(skill); - } - - /** - * 校验通过完整 Skill。 + * 完整标准 Skill 校验通过。 */ @Test public void validateCompleteSkill() { - Skill skill = validSkill(); - skill.getReferences().add(reference("references/a.md", "# A")); - skill.getScripts().add(script("scripts/run.sh", "echo ok")); - skill.getAssets().add(asset("assets/a.bin", "abc")); + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("references/a.md", SkillResourceKind.REFERENCE, "# A"), + textResource("scripts/run.sh", SkillResourceKind.SCRIPT, "echo ok"), + binaryResource("assets/a.bin", "abc") + )); validator.validate(skill); } - private static Skill validSkill() { - return SkillFactory.create("skill-a", "---\nname: Skill A\ndescription: Desc A\n---\n# Skill A\n"); + /** + * 嵌套 Map、List、布尔和数字 frontmatter 可校验。 + */ + @Test + public void allowNestedFrontmatter() { + String content = "---\nname: nested-skill\ndescription: Handles nested metadata\n" + + "metadata:\n enabled: true\n retries: 3\n tags:\n - alpha\n - beta\n" + + "---\n# Nested\n"; + Skill skill = SkillFactory.create("repository-id", content); + + validator.validate(skill); + + Assert.assertTrue(skill.getMetadata().get("metadata") instanceof java.util.Map); } - private static SkillReference reference(String path, String content) { - SkillReference reference = new SkillReference(); - reference.setPath(path); - reference.setName("a.md"); - reference.setContent(content); - reference.setContentHash(SkillHashes.sha256Hex(content.getBytes(StandardCharsets.UTF_8))); - reference.setSize(content.getBytes(StandardCharsets.UTF_8).length); - return reference; + /** + * 结构化文档重新应用到聚合时同步 name、description 和 metadata。 + */ + @Test + public void applyEditedDocumentToAggregate() { + Skill skill = validSkill("skill-a"); + com.easyagents.skill.model.SkillDocument document = skill.getDocument(); + document.putFrontmatter("description", "Updated description for the Skill"); + + skill.setDocument(document); + + Assert.assertEquals("Updated description for the Skill", skill.getDescription()); + Assert.assertEquals("Updated description for the Skill", skill.getMetadata().get("description")); + validator.validate(skill); } - private static SkillScript script(String path, String content) { - SkillScript script = new SkillScript(); - script.setPath(path); - script.setLanguage(SkillScriptLanguage.fromPath(path)); - script.setContent(content); - script.setContentHash(SkillHashes.sha256Hex(content.getBytes(StandardCharsets.UTF_8))); - script.setSize(content.getBytes(StandardCharsets.UTF_8).length); - return script; + /** + * 历史下划线名称只产生 warning。 + */ + @Test + public void legacyUnderscoreNameProducesWarning() { + Skill skill = validSkill("legacy_skill"); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertFalse(report.hasErrors()); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "NON_CANONICAL_NAME".equals(issue.getCode()) + && issue.getSeverity() == SkillValidationSeverity.WARNING)); } - private static SkillAsset asset(String path, String content) { + /** + * 正式标准模式拒绝仅为草稿导入兼容的下划线名称。 + */ + @Test + public void standardModeRejectsLegacyUnderscoreName() { + Skill skill = validSkill("legacy_skill"); + + SkillValidationReport report = validator.validateReport( + skill, SkillPackageLimits.defaults(), SkillValidationMode.STANDARD); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "NON_CANONICAL_NAME".equals(issue.getCode()) + && issue.getSeverity() == SkillValidationSeverity.ERROR)); + } + + /** + * 严格工厂入口执行正式标准名称校验。 + */ + @Test(expected = SkillValidationException.class) + public void strictFactoryRejectsLegacyUnderscoreName() { + SkillFactory.createStrict("repository-id", skillMd("legacy_skill")); + } + + /** + * AgentScope 嵌套 metadata 保真并标记标准兼容 warning。 + */ + @Test + public void nestedMetadataProducesCompatibilityWarning() { + Skill skill = SkillFactory.create("id", "---\nname: metadata-skill\n" + + "description: Nested metadata compatibility\nmetadata:\n provider:\n enabled: true\n" + + "---\n# Metadata\n"); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertFalse(report.hasErrors()); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "NON_STANDARD_METADATA".equals(issue.getCode()))); + } + + /** + * 标准可选字符串字段拒绝空值或非字符串。 + */ + @Test + public void rejectInvalidOptionalStandardFields() { + Skill skill = SkillFactory.create("id", "---\nname: optional-skill\n" + + "description: Invalid optional fields\nlicense: []\ncompatibility: ''\nallowed-tools:\n - Read\n" + + "---\n# Optional\n"); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "INVALID_LICENSE".equals(issue.getCode()) + && Integer.valueOf(4).equals(issue.getLine()) + && Integer.valueOf(1).equals(issue.getColumn()))); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "INVALID_COMPATIBILITY".equals(issue.getCode()) + && Integer.valueOf(5).equals(issue.getLine()) + && Integer.valueOf(1).equals(issue.getColumn()))); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "INVALID_ALLOWED_TOOLS".equals(issue.getCode()) + && Integer.valueOf(6).equals(issue.getLine()) + && Integer.valueOf(1).equals(issue.getColumn()))); + } + + /** + * 非规范名称返回结构化错误。 + */ + @Test + public void rejectInvalidName() { + Skill skill = validSkill("Invalid Name"); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.hasErrors()); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> "INVALID_NAME".equals(issue.getCode()))); + } + + /** + * 名称与目录不一致时失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectRootNameMismatch() { + Skill skill = validSkill("skill-a"); + skill.setPackageRoot("other-skill"); + + validator.validate(skill); + } + + /** + * 大小写冲突路径会被拒绝。 + */ + @Test + public void rejectCaseConflictingResourcePaths() { + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("references/A.md", SkillResourceKind.REFERENCE, "A"), + textResource("references/a.md", SkillResourceKind.REFERENCE, "B") + )); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "RESOURCE_PATH_COLLISION".equals(issue.getCode()))); + } + + /** + * 文件资源不能同时作为另一个资源的祖先路径。 + */ + @Test + public void rejectFileAndDescendantResourcePaths() { + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("references/a.md", SkillResourceKind.REFERENCE, "descendant"), + textResource("references", SkillResourceKind.OTHER, "file") + )); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "RESOURCE_PATH_HIERARCHY_CONFLICT".equals(issue.getCode()))); + } + + /** + * 路径穿越会被拒绝。 + */ + @Test + public void rejectParentPath() { + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("references/../a.md", SkillResourceKind.REFERENCE, "A"))); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "UNSAFE_RESOURCE_PATH".equals(issue.getCode()))); + } + + /** + * 缺失资源路径必须形成结构化问题,并继续聚合其他资源错误。 + */ + @Test + public void aggregateMissingPathWithOtherResourceErrors() { + Skill skill = validSkill("skill-a"); + SkillResource missingPath = textResource( + "references/missing.md", SkillResourceKind.REFERENCE, "missing"); + missingPath.setPath(null); + skill.setResources(java.util.List.of( + missingPath, + textResource("references/../escape.md", SkillResourceKind.REFERENCE, "escape"))); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "RESOURCE_PATH_REQUIRED".equals(issue.getCode()) && issue.getPath() == null)); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "UNSAFE_RESOURCE_PATH".equals(issue.getCode()))); + } + + /** + * 文本资源 hash 和 size 必须与 UTF-8 内容一致。 + */ + @Test + public void rejectTextHashAndSizeMismatch() { + Skill skill = validSkill("skill-a"); + SkillResource resource = textResource("references/a.md", SkillResourceKind.REFERENCE, "A"); + resource.setContentHash("0".repeat(64)); + resource.setSize(99); + skill.setResources(java.util.List.of(resource)); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "RESOURCE_HASH_MISMATCH".equals(issue.getCode()))); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "RESOURCE_SIZE_MISMATCH".equals(issue.getCode()))); + } + + /** + * scripts 目录只接受严格 UTF-8 文本表示,二进制引用必须被拒绝。 + */ + @Test + public void rejectBinaryScriptResource() { + Skill skill = validSkill("skill-a"); + SkillResource script = binaryResource("scripts/run.sh", "echo unsafe"); + script.setKind(SkillResourceKind.SCRIPT); + skill.setResources(java.util.List.of(script)); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "SCRIPT_TEXT_REQUIRED".equals(issue.getCode()) + && issue.getSeverity() == SkillValidationSeverity.ERROR)); + } + + /** + * 资源的文本或二进制表示必须与统一路径和媒体类型判定一致。 + */ + @Test + public void rejectNonCanonicalResourceContentMode() { + Skill skill = validSkill("skill-a"); + SkillResource textAsset = textResource("assets/readme.txt", SkillResourceKind.ASSET, "text asset"); + skill.setResources(java.util.List.of(textAsset)); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "RESOURCE_CONTENT_MODE_MISMATCH".equals(issue.getCode()))); + } + + /** + * 空脚本允许保存在草稿中,但应返回可见 warning。 + */ + @Test + public void warnForEmptyScriptResource() { + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("scripts/run.py", SkillResourceKind.SCRIPT, ""))); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertFalse(report.hasErrors()); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "EMPTY_SCRIPT".equals(issue.getCode()) + && issue.getSeverity() == SkillValidationSeverity.WARNING)); + } + + /** + * 未识别脚本语言需要提示,但不能破坏外部标准 Skill 包的资源保真。 + */ + @Test + public void warnForUnrecognizedScriptLanguageExtension() { + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("scripts/run.txt", SkillResourceKind.SCRIPT, "echo ok"))); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertFalse(report.hasErrors()); + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "SCRIPT_LANGUAGE_UNRECOGNIZED".equals(issue.getCode()) + && issue.getSeverity() == SkillValidationSeverity.WARNING)); + } + + /** + * 结构化校验器执行资源单文件安全限额。 + */ + @Test + public void reportConfiguredResourceLimit() { + Skill skill = validSkill("skill-a"); + skill.setResources(java.util.List.of( + textResource("references/a.md", SkillResourceKind.REFERENCE, "x".repeat(300)))); + DefaultSkillValidator limitedValidator = new DefaultSkillValidator( + SkillPackageLimits.builder().maxTextFileBytes(128).build()); + + SkillValidationReport report = limitedValidator.validateReport(skill); + + Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> + "TEXT_FILE_SIZE_LIMIT".equals(issue.getCode()))); + } + + /** + * 单次调用传入的限额优先于校验器构造时的默认限额。 + */ + @Test + public void operationLimitsOverrideValidatorDefaults() { + Skill skill = validSkill("skill-a"); + String path = "references/" + "a".repeat(520) + ".md"; + skill.setResources(java.util.List.of( + textResource(path, SkillResourceKind.REFERENCE, "A"))); + SkillPackageLimits operationLimits = SkillPackageLimits.builder() + .maxPathLength(600) + .build(); + + SkillValidationReport report = validator.validateReport(skill, operationLimits); + + Assert.assertFalse(report.hasErrors()); + } + + /** + * 元数据与 SKILL.md 不一致时失败。 + */ + @Test(expected = SkillValidationException.class) + public void rejectMetadataMismatch() { + Skill skill = validSkill("skill-a"); + skill.getMetadata().put("extra", "value"); + + validator.validate(skill); + } + + private static Skill validSkill(String name) { + return SkillFactory.create("repository-id", skillMd(name)); + } + + private static String skillMd(String name) { + return "---\nname: " + name + "\ndescription: Use this Skill for validator tests\n---\n# Skill\n"; + } + + private static SkillResource textResource(String path, SkillResourceKind kind, String content) { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + SkillResource resource = new SkillResource(); + resource.setPath(path); + resource.setKind(kind); + resource.setMediaType("text/markdown"); + resource.setTextContent(content); + resource.setContentHash(SkillHashes.sha256Hex(bytes)); + resource.setSize(bytes.length); + return resource; + } + + private static SkillResource binaryResource(String path, String content) { byte[] bytes = content.getBytes(StandardCharsets.UTF_8); String hash = SkillHashes.sha256Hex(bytes); - SkillAsset asset = new SkillAsset(); - asset.setPath(path); - asset.setName("a.bin"); - asset.setMediaType("application/octet-stream"); - asset.setContentRef("sha256:" + hash); - asset.setContentHash(hash); - asset.setSize(bytes.length); - return asset; + SkillResource resource = new SkillResource(); + resource.setPath(path); + resource.setKind(SkillResourceKind.ASSET); + resource.setMediaType("application/octet-stream"); + resource.setContentRef("sha256:" + hash); + resource.setContentHash(hash); + resource.setSize(bytes.length); + return resource; } } diff --git a/pom.xml b/pom.xml index 28b5090..455abe1 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,8 @@ 1.8.22 1.51.0 1.0.12 + 2.6 + 1.28.0 @@ -114,6 +116,18 @@ ${junit.version} + + org.yaml + snakeyaml + ${snakeyaml.version} + + + + org.apache.commons + commons-compress + ${commons-compress.version} + + com.easyagents From a7e89cee3dfccf3186a84184e1927b5926c24387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Tue, 28 Jul 2026 12:24:38 +0800 Subject: [PATCH 11/33] =?UTF-8?q?fix:=20=E7=BB=9F=E4=B8=80=20OpenAI=20?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E5=86=85=E5=AE=B9=E5=9D=97=E6=95=B0=E7=BB=84?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将 system、user、assistant、tool 及历史上下文 content 统一为数组 - 保留 DeepSeek、GLM 专用格式规则并补充多模态与工具消息测试 --- .../AgentDeepSeekChatFormatter.java | 24 +++ .../agentscope/AgentGLMChatFormatter.java | 24 +++ .../agentscope/AgentOpenAIChatFormatter.java | 23 ++- .../agentscope/AgentScopeModelFactory.java | 12 +- .../model/AgentMessageContentFormat.java | 13 ++ .../agent/runtime/model/AgentModelSpec.java | 22 +-- .../model/AgentSystemContentFormat.java | 13 -- .../AgentOpenAIChatFormatterTest.java | 169 +++++++++++++++++- 8 files changed, 259 insertions(+), 41 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentDeepSeekChatFormatter.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentGLMChatFormatter.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentMessageContentFormat.java delete mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentDeepSeekChatFormatter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentDeepSeekChatFormatter.java new file mode 100644 index 0000000..6736328 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentDeepSeekChatFormatter.java @@ -0,0 +1,24 @@ +package com.easyagents.agent.runtime.agentscope; + +import io.agentscope.core.formatter.openai.DeepSeekFormatter; +import io.agentscope.core.formatter.openai.dto.OpenAIMessage; +import io.agentscope.core.message.Msg; + +import java.util.List; + +/** + * 保留 DeepSeek 专用规则并将全部消息 content 规范为内容块数组。 + */ +public final class AgentDeepSeekChatFormatter extends DeepSeekFormatter { + + /** + * 转换 DeepSeek 消息并在供应商规则之后统一 content 格式。 + * + * @param messages AgentScope 消息 + * @return OpenAI 请求消息 + */ + @Override + protected List doFormat(List messages) { + return AgentOpenAIChatFormatter.normalizeContent(super.doFormat(messages)); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentGLMChatFormatter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentGLMChatFormatter.java new file mode 100644 index 0000000..6937ebb --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentGLMChatFormatter.java @@ -0,0 +1,24 @@ +package com.easyagents.agent.runtime.agentscope; + +import io.agentscope.core.formatter.openai.GLMFormatter; +import io.agentscope.core.formatter.openai.dto.OpenAIMessage; +import io.agentscope.core.message.Msg; + +import java.util.List; + +/** + * 保留 GLM 专用规则并将全部消息 content 规范为内容块数组。 + */ +public final class AgentGLMChatFormatter extends GLMFormatter { + + /** + * 转换 GLM 消息并在供应商规则之后统一 content 格式。 + * + * @param messages AgentScope 消息 + * @return OpenAI 请求消息 + */ + @Override + protected List doFormat(List messages) { + return AgentOpenAIChatFormatter.normalizeContent(super.doFormat(messages)); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java index 8f56602..da0595d 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatter.java @@ -8,22 +8,37 @@ import io.agentscope.core.message.Msg; import java.util.List; /** - * 为 OpenAI-compatible 模型补充 system content 内容块数组兼容能力。 + * 为 OpenAI-compatible 模型补充全部消息 content 内容块数组兼容能力。 */ public final class AgentOpenAIChatFormatter extends OpenAIChatFormatter { /** - * 将 AgentScope 消息转换为 OpenAI 消息,并规范 system 文本的 content 格式。 + * 将 AgentScope 消息转换为 OpenAI 消息,并规范全部角色的 content 格式。 * * @param messages AgentScope 消息 * @return OpenAI 请求消息 */ @Override protected List doFormat(List messages) { - List formattedMessages = super.doFormat(messages); + return normalizeContent(super.doFormat(messages)); + } + + /** + * 将 OpenAI 消息中的 content 统一规范为内容块数组。 + * + * @param formattedMessages 已完成供应商规则转换的 OpenAI 消息 + * @return content 已规范为数组的原消息列表 + */ + static List normalizeContent(List formattedMessages) { for (OpenAIMessage message : formattedMessages) { - if ("system".equals(message.getRole()) && message.getContent() instanceof String text) { + Object content = message.getContent(); + if (content instanceof String text) { message.setContent(List.of(OpenAIContentPart.text(text))); + } else if (content == null) { + message.setContent(List.of(OpenAIContentPart.text(""))); + } else if (!(content instanceof List)) { + throw new IllegalStateException( + "Unsupported OpenAI message content type: " + content.getClass().getName()); } } return formattedMessages; diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java index 2f5ccdb..9002e6e 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeModelFactory.java @@ -6,7 +6,7 @@ import com.easyagents.agent.runtime.model.AgentHttpVersionPolicy; import com.easyagents.agent.runtime.model.AgentModelFactory; import com.easyagents.agent.runtime.model.AgentModelProviderType; import com.easyagents.agent.runtime.model.AgentModelSpec; -import com.easyagents.agent.runtime.model.AgentSystemContentFormat; +import com.easyagents.agent.runtime.model.AgentMessageContentFormat; import io.agentscope.core.formatter.openai.DeepSeekFormatter; import io.agentscope.core.formatter.openai.GLMFormatter; import io.agentscope.core.model.*; @@ -154,7 +154,7 @@ public class AgentScopeModelFactory implements AgentModelFactory { .stream(Boolean.TRUE.equals(options.getStream())) .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) .generateOptions(options); - if (modelSpec.getSystemContentFormat() == AgentSystemContentFormat.TEXT_PARTS) { + if (modelSpec.getMessageContentFormat() == AgentMessageContentFormat.TEXT_PARTS) { builder.formatter(new AgentOpenAIChatFormatter()); } return builder.build(); @@ -210,7 +210,9 @@ public class AgentScopeModelFactory implements AgentModelFactory { .endpointPath(modelSpec.getEndpointPath()) .stream(Boolean.TRUE.equals(options.getStream())) .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) - .formatter(new DeepSeekFormatter()) + .formatter(modelSpec.getMessageContentFormat() == AgentMessageContentFormat.TEXT_PARTS + ? new AgentDeepSeekChatFormatter() + : new DeepSeekFormatter()) .generateOptions(options); return builder.build(); } @@ -231,7 +233,9 @@ public class AgentScopeModelFactory implements AgentModelFactory { .endpointPath(modelSpec.getEndpointPath()) .stream(Boolean.TRUE.equals(options.getStream())) .httpTransport(httpTransportProvider.getTransport(modelSpec.getHttpVersionPolicy(), baseUrl)) - .formatter(new GLMFormatter()) + .formatter(modelSpec.getMessageContentFormat() == AgentMessageContentFormat.TEXT_PARTS + ? new AgentGLMChatFormatter() + : new GLMFormatter()) .generateOptions(options); return builder.build(); } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentMessageContentFormat.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentMessageContentFormat.java new file mode 100644 index 0000000..0441f57 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentMessageContentFormat.java @@ -0,0 +1,13 @@ +package com.easyagents.agent.runtime.model; + +/** + * OpenAI-compatible 请求中消息 content 的格式策略。 + */ +public enum AgentMessageContentFormat { + + /** 使用 AgentScope 默认格式,纯文本为字符串,多模态内容为数组。 */ + STANDARD, + + /** 将全部角色的 content 规范为内容块数组。 */ + TEXT_PARTS +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java index ac33ff1..9b3d1cd 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentModelSpec.java @@ -16,7 +16,7 @@ public class AgentModelSpec { private boolean supportImage; private boolean supportImageBase64Only; private AgentHttpVersionPolicy httpVersionPolicy = AgentHttpVersionPolicy.AUTO; - private AgentSystemContentFormat systemContentFormat = AgentSystemContentFormat.STRING; + private AgentMessageContentFormat messageContentFormat = AgentMessageContentFormat.STANDARD; private Map metadata = new LinkedHashMap<>(); /** @@ -164,23 +164,23 @@ public class AgentModelSpec { } /** - * 获取 OpenAI-compatible 请求中的 system content 格式。 + * 获取 OpenAI-compatible 请求中的消息 content 格式。 * - * @return system content 格式 + * @return 消息 content 格式 */ - public AgentSystemContentFormat getSystemContentFormat() { - return systemContentFormat; + public AgentMessageContentFormat getMessageContentFormat() { + return messageContentFormat; } /** - * 设置 OpenAI-compatible 请求中的 system content 格式。 + * 设置 OpenAI-compatible 请求中的消息 content 格式。 * - * @param systemContentFormat system content 格式 + * @param messageContentFormat 消息 content 格式 */ - public void setSystemContentFormat(AgentSystemContentFormat systemContentFormat) { - this.systemContentFormat = systemContentFormat == null - ? AgentSystemContentFormat.STRING - : systemContentFormat; + public void setMessageContentFormat(AgentMessageContentFormat messageContentFormat) { + this.messageContentFormat = messageContentFormat == null + ? AgentMessageContentFormat.STANDARD + : messageContentFormat; } /** diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java deleted file mode 100644 index f06fd45..0000000 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/model/AgentSystemContentFormat.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.easyagents.agent.runtime.model; - -/** - * OpenAI-compatible 请求中 system 消息的 content 格式。 - */ -public enum AgentSystemContentFormat { - - /** 使用字符串 content,保持 OpenAI Chat Completions 的常规格式。 */ - STRING, - - /** 使用仅包含 text 内容块的数组 content。 */ - TEXT_PARTS -} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java index 3e4816e..18a1533 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentOpenAIChatFormatterTest.java @@ -1,19 +1,25 @@ package com.easyagents.agent.runtime.agentscope; import com.easyagents.agent.runtime.model.AgentGenerationOptions; +import com.easyagents.agent.runtime.model.AgentMessageContentFormat; import com.easyagents.agent.runtime.model.AgentModelProviderType; import com.easyagents.agent.runtime.model.AgentModelSpec; -import com.easyagents.agent.runtime.model.AgentSystemContentFormat; import io.agentscope.core.formatter.openai.dto.OpenAIContentPart; import io.agentscope.core.formatter.openai.dto.OpenAIMessage; +import io.agentscope.core.message.Base64Source; +import io.agentscope.core.message.ImageBlock; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.message.ToolUseBlock; import io.agentscope.core.model.OpenAIChatModel; import org.junit.Assert; import org.junit.Test; import java.lang.reflect.Field; import java.util.List; +import java.util.Map; /** * Agent OpenAI Chat 消息格式兼容测试。 @@ -21,21 +27,96 @@ import java.util.List; public class AgentOpenAIChatFormatterTest { /** - * 验证 system 文本转换为 text 内容块数组,普通 user 文本保持字符串。 + * 验证多轮上下文中全部纯文本消息都转换为 text 内容块数组。 */ @Test - public void shouldConvertOnlySystemTextToContentParts() { + public void shouldConvertAllTextMessagesToContentParts() { AgentOpenAIChatFormatter formatter = new AgentOpenAIChatFormatter(); List messages = formatter.format(List.of( Msg.builder().role(MsgRole.SYSTEM).textContent("system prompt").build(), - Msg.builder().role(MsgRole.USER).textContent("hello").build())); + Msg.builder().role(MsgRole.USER).textContent("hello").build(), + Msg.builder().role(MsgRole.ASSISTANT).textContent("hi").build(), + Msg.builder().role(MsgRole.USER).textContent("follow up").build())); + + assertTextContent(messages.get(0), "system prompt"); + assertTextContent(messages.get(1), "hello"); + assertTextContent(messages.get(2), "hi"); + assertTextContent(messages.get(3), "follow up"); + } + + /** + * 验证图片加文本消息保持已有内容块数组和顺序。 + */ + @Test + public void shouldPreserveMultimodalContentParts() { + AgentOpenAIChatFormatter formatter = new AgentOpenAIChatFormatter(); + ImageBlock image = ImageBlock.builder() + .source(Base64Source.builder() + .mediaType("image/png") + .data("aW1hZ2U=") + .build()) + .build(); + + List messages = formatter.format(List.of( + Msg.builder() + .role(MsgRole.USER) + .content(TextBlock.builder().text("describe").build(), image) + .build())); - Assert.assertTrue(messages.get(0).getContent() instanceof List); List contentParts = messages.get(0).getContentAsList(); - Assert.assertEquals(1, contentParts.size()); + Assert.assertNotNull(contentParts); + Assert.assertEquals(2, contentParts.size()); Assert.assertEquals("text", contentParts.get(0).getType()); - Assert.assertEquals("system prompt", contentParts.get(0).getText()); - Assert.assertEquals("hello", messages.get(1).getContent()); + Assert.assertEquals("describe", contentParts.get(0).getText()); + Assert.assertEquals("image_url", contentParts.get(1).getType()); + } + + /** + * 验证工具调用和工具结果的附属字段在 content 数组化后保持不变。 + */ + @Test + public void shouldPreserveToolCallFieldsAndConvertToolResult() { + AgentOpenAIChatFormatter formatter = new AgentOpenAIChatFormatter(); + ToolUseBlock toolUse = ToolUseBlock.builder() + .id("call-1") + .name("lookup") + .input(Map.of("query", "weather")) + .build(); + ToolResultBlock toolResult = ToolResultBlock.builder() + .id("call-1") + .name("lookup") + .output(TextBlock.builder().text("sunny").build()) + .build(); + + List messages = formatter.format(List.of( + Msg.builder().role(MsgRole.ASSISTANT).content(toolUse).build(), + Msg.builder().role(MsgRole.TOOL).content(toolResult).build())); + + assertTextContent(messages.get(0), ""); + Assert.assertNotNull(messages.get(0).getToolCalls()); + Assert.assertEquals(1, messages.get(0).getToolCalls().size()); + Assert.assertEquals("call-1", messages.get(0).getToolCalls().get(0).getId()); + assertTextContent(messages.get(1), "sunny"); + Assert.assertEquals("call-1", messages.get(1).getToolCallId()); + } + + /** + * 验证专用 Formatter 的供应商规则执行后仍会统一 content 数组。 + */ + @Test + public void shouldPreserveDeepSeekAndGlmRulesWithContentParts() { + List deepSeekMessages = new AgentDeepSeekChatFormatter().format(List.of( + Msg.builder().role(MsgRole.SYSTEM).textContent("system prompt").build())); + Assert.assertEquals("user", deepSeekMessages.get(0).getRole()); + assertTextContent(deepSeekMessages.get(0), "system prompt"); + + List glmMessages = new AgentGLMChatFormatter().format(List.of( + Msg.builder().role(MsgRole.SYSTEM).textContent("system prompt").build())); + Assert.assertEquals(2, glmMessages.size()); + Assert.assertEquals("system", glmMessages.get(0).getRole()); + Assert.assertEquals("user", glmMessages.get(1).getRole()); + assertTextContent(glmMessages.get(0), "system prompt"); + assertTextContent(glmMessages.get(1), ""); } /** @@ -50,7 +131,7 @@ public class AgentOpenAIChatFormatterTest { spec.setModelName("vlm-test"); spec.setBaseUrl("http://model.example.com/v1"); spec.setApiKey("test-key"); - spec.setSystemContentFormat(AgentSystemContentFormat.TEXT_PARTS); + spec.setMessageContentFormat(AgentMessageContentFormat.TEXT_PARTS); OpenAIChatModel model = (OpenAIChatModel) new AgentScopeModelFactory() .create(spec, new AgentGenerationOptions()); @@ -59,4 +140,74 @@ public class AgentOpenAIChatFormatterTest { Assert.assertTrue(formatterField.get(model) instanceof AgentOpenAIChatFormatter); } + + /** + * 验证默认策略继续使用 AgentScope 原生 Formatter。 + * + * @throws Exception 反射读取 Formatter 失败时抛出 + */ + @Test + public void standardPolicyShouldKeepDefaultOpenAIFormatter() throws Exception { + AgentModelSpec spec = new AgentModelSpec(); + spec.setProviderType(AgentModelProviderType.OPENAI_COMPATIBLE); + spec.setModelName("chat-test"); + spec.setBaseUrl("http://model.example.com/v1"); + spec.setApiKey("test-key"); + + OpenAIChatModel model = (OpenAIChatModel) new AgentScopeModelFactory() + .create(spec, new AgentGenerationOptions()); + Field formatterField = OpenAIChatModel.class.getDeclaredField("formatter"); + formatterField.setAccessible(true); + + Assert.assertFalse(formatterField.get(model) instanceof AgentOpenAIChatFormatter); + } + + /** + * 验证严格数组策略为专用 Provider 安装保留供应商规则的 Formatter。 + * + * @throws Exception 反射读取 Formatter 失败时抛出 + */ + @Test + public void textPartsPolicyShouldInstallProviderSpecificFormatters() throws Exception { + Assert.assertTrue(formatterFor(AgentModelProviderType.DEEPSEEK) + instanceof AgentDeepSeekChatFormatter); + Assert.assertTrue(formatterFor(AgentModelProviderType.GLM) + instanceof AgentGLMChatFormatter); + } + + /** + * 创建指定 Provider 的严格数组模型并读取其 Formatter。 + * + * @param providerType Provider 类型 + * @return 模型 Formatter + * @throws Exception 反射读取 Formatter 失败时抛出 + */ + private Object formatterFor(AgentModelProviderType providerType) throws Exception { + AgentModelSpec spec = new AgentModelSpec(); + spec.setProviderType(providerType); + spec.setModelName("provider-test"); + spec.setApiKey("test-key"); + spec.setMessageContentFormat(AgentMessageContentFormat.TEXT_PARTS); + + OpenAIChatModel model = (OpenAIChatModel) new AgentScopeModelFactory() + .create(spec, new AgentGenerationOptions()); + Field formatterField = OpenAIChatModel.class.getDeclaredField("formatter"); + formatterField.setAccessible(true); + return formatterField.get(model); + } + + /** + * 断言消息只包含一个指定文本内容块。 + * + * @param message OpenAI 请求消息 + * @param expectedText 预期文本 + */ + private void assertTextContent(OpenAIMessage message, String expectedText) { + Assert.assertTrue(message.getContent() instanceof List); + List contentParts = message.getContentAsList(); + Assert.assertNotNull(contentParts); + Assert.assertEquals(1, contentParts.size()); + Assert.assertEquals("text", contentParts.get(0).getType()); + Assert.assertEquals(expectedText, contentParts.get(0).getText()); + } } From c72a167633834bfc1cc732a77b6b171ec94904aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 29 Jul 2026 00:47:41 +0800 Subject: [PATCH 12/33] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E6=B5=81=E6=89=A7=E8=A1=8C=E5=BC=95=E6=93=8E=E4=B8=8E?= =?UTF-8?q?=E5=BE=AA=E7=8E=AF=E8=B0=83=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 引入增量状态、定义快照和持久化触发器 - 收敛循环结果、模板条件和高 IO 执行开销 - 统一循环 1 至 300 次约束并补充并发回归测试 --- .../com/easyagents/flow/core/chain/Chain.java | 1420 +++++++++++++++-- .../flow/core/chain/ChainDefinition.java | 334 +++- .../flow/core/chain/ChainState.java | 387 ++++- .../com/easyagents/flow/core/chain/Edge.java | 9 +- .../flow/core/chain/EdgeCondition.java | 14 +- .../flow/core/chain/ExceptionSummary.java | 2 +- .../com/easyagents/flow/core/chain/Node.java | 29 +- .../flow/core/chain/NodeCondition.java | 14 +- .../easyagents/flow/core/chain/NodeState.java | 47 +- .../flow/core/chain/NodeValidator.java | 14 +- .../easyagents/flow/core/chain/Parameter.java | 1 + .../flow/core/chain/event/NodeEndEvent.java | 93 ++ .../flow/core/chain/event/NodeStartEvent.java | 102 +- .../ChainDefinitionSnapshotRepository.java | 36 + .../flow/core/chain/repository/ChainLock.java | 20 +- .../chain/repository/ChainStateField.java | 6 +- .../repository/ChainStateRepository.java | 62 + ...moryChainDefinitionSnapshotRepository.java | 43 + .../InMemoryChainStateRepository.java | 21 +- .../InMemoryLoopResultRepository.java | 111 ++ .../InMemoryNodeStateRepository.java | 35 +- .../chain/repository/LoopInputReference.java | 62 + .../chain/repository/LoopResultReference.java | 52 + .../repository/LoopResultRepository.java | 486 ++++++ .../core/chain/repository/NodeStateField.java | 4 +- .../chain/repository/NodeStateRepository.java | 116 ++ .../core/chain/runtime/ChainExecutor.java | 841 +++++++++- .../core/chain/runtime/ExecutionBudget.java | 188 +++ .../ExecutionBudgetExceededException.java | 18 + .../chain/runtime/InMemoryTriggerStore.java | 42 +- .../runtime/NonRetryableTriggerException.java | 16 + .../runtime/RetryableTriggerException.java | 17 + .../flow/core/chain/runtime/Trigger.java | 306 +++- .../runtime/TriggerClaimLostException.java | 20 + .../core/chain/runtime/TriggerScheduler.java | 729 ++++++++- .../flow/core/chain/runtime/TriggerStore.java | 131 ++ .../code/impl/JavascriptRuntimeEngine.java | 18 +- .../com/easyagents/flow/core/llm/Llm.java | 1 + .../easyagents/flow/core/node/BaseNode.java | 1 + .../easyagents/flow/core/node/CodeNode.java | 5 +- .../flow/core/node/ConfirmNode.java | 12 +- .../easyagents/flow/core/node/EndNode.java | 9 +- .../easyagents/flow/core/node/HttpNode.java | 256 ++- .../flow/core/node/KnowledgeNode.java | 14 +- .../easyagents/flow/core/node/LlmNode.java | 19 +- .../easyagents/flow/core/node/LoopNode.java | 794 ++++++++- .../flow/core/node/SearchEngineNode.java | 14 +- .../easyagents/flow/core/node/StartNode.java | 5 +- .../flow/core/node/TemplateNode.java | 5 +- .../flow/core/parser/BaseNodeParser.java | 32 +- .../easyagents/flow/core/util/IoBulkhead.java | 489 ++++++ .../flow/core/util/IterableUtil.java | 22 + .../flow/core/util/JsConditionUtil.java | 109 +- .../flow/core/util/OkHttpClientUtil.java | 188 ++- .../flow/core/util/TextTemplate.java | 213 ++- .../node/HttpNodePerformanceSafetyTest.java | 126 ++ .../core/test/ChainDefinitionIndexTest.java | 128 ++ .../test/ChainExecutorConcurrencyTest.java | 455 ++++++ .../core/test/ChainRecoverableStartTest.java | 313 ++++ .../core/test/ChainTemplateContextTest.java | 231 +++ .../flow/core/test/ExecutionBudgetTest.java | 63 + .../core/test/GenericNodeLoopCountTest.java | 325 ++++ .../flow/core/test/IoBulkheadTest.java | 82 + .../flow/core/test/JsConditionUtilTest.java | 91 ++ .../LegacySerializationCompatibilityTest.java | 83 + .../test/LoopNodeProgressContextTest.java | 605 ++++++- .../test/LoopResultReferenceResolverTest.java | 123 ++ .../flow/core/test/OkHttpClientUtilTest.java | 116 ++ .../flow/core/test/TextTemplatePathTest.java | 62 + .../test/TriggerSchedulerReliabilityTest.java | 973 +++++++++++ 70 files changed, 11338 insertions(+), 472 deletions(-) create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainDefinitionSnapshotRepository.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainDefinitionSnapshotRepository.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryLoopResultRepository.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopInputReference.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudget.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudgetExceededException.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/NonRetryableTriggerException.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/RetryableTriggerException.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerClaimLostException.java create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/util/IoBulkhead.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/node/HttpNodePerformanceSafetyTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainDefinitionIndexTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainRecoverableStartTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ExecutionBudgetTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/GenericNodeLoopCountTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/IoBulkheadTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JsConditionUtilTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LegacySerializationCompatibilityTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/OkHttpClientUtilTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TriggerSchedulerReliabilityTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java index be2fe97..df5c58e 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java @@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -33,6 +34,14 @@ public class Chain { private static final Logger log = LoggerFactory.getLogger(Chain.class); private static final ThreadLocal EXECUTION_THREAD_LOCAL = new ThreadLocal<>(); + /** + * 当前节点执行期间的只读状态视图;线程隔离避免并行节点互相覆盖。 + */ + private static final ThreadLocal + NODE_EXECUTION_STATE_VIEW = new ThreadLocal<>(); + private static final ThreadLocal> HELD_INSTANCE_LOCKS = + new ThreadLocal<>(); + private static final ThreadLocal> DEFERRED_EVENT_ACTIONS = new ThreadLocal<>(); protected final ChainDefinition definition; @@ -41,8 +50,23 @@ public class Chain { // protected final ChainState state; protected ChainStateRepository chainStateRepository; protected NodeStateRepository nodeStateRepository; + protected LoopResultRepository loopResultRepository = new InMemoryLoopResultRepository(); protected EventManager eventManager; protected TriggerScheduler triggerScheduler; + /** + * 当前实例锁内已经读取或成功提交的状态快照。 + */ + private ChainState executionStateSnapshot; + /** + * 产生当前状态快照的实例锁;锁作用域变化后快照不得继续用于判断或修改。 + */ + private ChainLock executionStateSnapshotLock; + /** + * 当前执行实例使用的平台资源保护预算。 + */ + protected ExecutionBudget executionBudget = ExecutionBudget.defaults(); + protected String executionLane; + protected int nestedDepthBase; public static Chain currentChain() { return EXECUTION_THREAD_LOCAL.get(); @@ -54,7 +78,7 @@ public class Chain { } public void notifyEvent(Event event) { - eventManager.notifyEvent(event, this); + deferOrRun(() -> eventManager.notifyEvent(event, this)); } public void setStatusAndNotifyEvent(ChainStatus status) { @@ -89,6 +113,18 @@ public class Chain { public ChainState updateStateSafely(String stateInstanceId, ChainStateModifier modifier) { + return executeWithLock(stateInstanceId, 10, TimeUnit.SECONDS, + () -> updateStateSafelyLocked(stateInstanceId, modifier)); + } + + /** + * 在实例锁保护下更新工作流状态。 + * + * @param stateInstanceId 工作流实例 ID + * @param modifier 状态修改器 + * @return 更新后的状态 + */ + private ChainState updateStateSafelyLocked(String stateInstanceId, ChainStateModifier modifier) { final long timeoutMs = 30_000; // 30 seconds total timeout final long maxRetryDelayMs = 100; // Maximum delay between retries @@ -96,19 +132,41 @@ public class Chain { int attempt = 0; ChainState current = null; while (System.currentTimeMillis() - startTime < timeoutMs) { - current = chainStateRepository.load(stateInstanceId); + current = attempt == 0 + ? reusableExecutionState(stateInstanceId) + : null; + if (current == null) { + current = chainStateRepository.load(stateInstanceId); + } if (current == null) { throw new IllegalStateException("Chain state not found: " + stateInstanceId); } EnumSet updatedFields = modifier.modify(current); if (updatedFields == null || updatedFields.isEmpty()) { + cacheExecutionState(stateInstanceId, current); return current; // No actual changes, exit early } - if (chainStateRepository.tryUpdate(current, updatedFields)) { + current.setVersion(current.getVersion() + 1); + updatedFields.add(ChainStateField.VERSION); + executionBudget.checkHotStateBytes(estimateHotStateBytes(current)); + assertInstanceLockOwned(stateInstanceId); + if (chainStateRepository.tryUpdate( + current, + updatedFields, + currentLockFencingToken(stateInstanceId), + currentFencingClaimId(stateInstanceId), + currentClaimGeneration(stateInstanceId))) { + cacheExecutionState(stateInstanceId, current); return current; } + assertTriggerClaimOwned(TriggerContext.getCurrentTrigger()); + if (currentLockFencingToken(stateInstanceId) > 0L) { + throw new RetryableTriggerException( + "Workflow state commit guard rejected: " + stateInstanceId, + null); + } // Prepare next retry attempt++; @@ -132,24 +190,47 @@ public class Chain { } public NodeState updateNodeStateSafely(String stateInstanceId, String nodeId, NodeStateModifier modifier) { + return executeWithLock(stateInstanceId, 10, TimeUnit.SECONDS, + () -> updateNodeStateSafelyLocked(stateInstanceId, nodeId, modifier)); + } + + /** + * 在实例锁保护下更新节点状态。 + * + * @param stateInstanceId 工作流实例 ID + * @param nodeId 节点 ID + * @param modifier 状态修改器 + * @return 更新后的节点状态 + */ + private NodeState updateNodeStateSafelyLocked(String stateInstanceId, + String nodeId, + NodeStateModifier modifier) { final long timeoutMs = 30_000; final long maxRetryDelayMs = 100; long startTime = System.currentTimeMillis(); int attempt = 0; while (System.currentTimeMillis() - startTime < timeoutMs) { - // 1. 加载最新 ChainState(获取 chainVersion) - ChainState chainState = chainStateRepository.load(stateInstanceId); - if (chainState == null) { + // 1. 仅加载 ChainState 版本,避免为节点状态提交反序列化完整工作流热状态。 + Long chainStateVersion = chainStateRepository.loadVersion(stateInstanceId); + if (chainStateVersion == null) { throw new IllegalStateException("Chain state not found"); } // 2. 加载 NodeState NodeState nodeState = nodeStateRepository.load(stateInstanceId, nodeId); if (nodeState == null) { - nodeState = new NodeState(); - nodeState.setChainInstanceId(chainState.getInstanceId()); - nodeState.setNodeId(nodeId); + nodeState = nodeStateRepository.create( + stateInstanceId, + nodeId, + chainStateVersion, + currentLockFencingToken(stateInstanceId), + currentFencingClaimId(stateInstanceId), + currentClaimGeneration(stateInstanceId)); + if (nodeState == null) { + throw new IllegalStateException( + "Unable to initialize node state: " + stateInstanceId + "/" + nodeId); + } } // 3. 应用修改 @@ -160,9 +241,34 @@ public class Chain { } // 4. 尝试更新(传入 chainVersion 保证一致性) - if (nodeStateRepository.tryUpdate(nodeState, updatedFields, chainState.getVersion())) { + nodeState.setVersion(nodeState.getVersion() + 1); + updatedFields.add(NodeStateField.VERSION); + executionBudget.checkHotStateBytes(estimateValueBytes( + Arrays.asList( + nodeState.getMemory(), + nodeState.getTriggerEdgeIds(), + nodeState.getExecuteEdgeIds()), + new IdentityHashMap<>(), + executionBudget.getMaxHotStateBytes())); + assertInstanceLockOwned(stateInstanceId); + if (nodeStateRepository.tryUpdate( + nodeState, + updatedFields, + chainStateVersion, + currentLockFencingToken(stateInstanceId), + currentFencingClaimId(stateInstanceId), + currentClaimGeneration(stateInstanceId))) { return nodeState; } + assertTriggerClaimOwned(TriggerContext.getCurrentTrigger()); + if (currentLockFencingToken(stateInstanceId) > 0L) { + throw new RetryableTriggerException( + "Workflow node state commit guard rejected: " + + stateInstanceId + + "/" + + nodeId, + null); + } // 5. 退避重试 attempt++; @@ -207,6 +313,87 @@ public class Chain { } } + /** + * 以短路方式估算热状态体积,避免为保护预算额外进行完整序列化。 + * + * @param state 工作流状态 + * @return 估算字节数 + */ + private long estimateHotStateBytes(ChainState state) { + long limit = executionBudget.getMaxHotStateBytes(); + if (limit <= 0) { + return 0L; + } + return estimateValueBytes(Arrays.asList( + state.getMemory(), + state.getExecuteResult(), + state.getTriggerEdgeIds(), + state.getTriggerNodeIds()), new IdentityHashMap<>(), limit); + } + + /** + * 递归估算对象图大小,到达预算后立即短路。 + * + * @param value 当前对象 + * @param visited 已访问对象 + * @param limit 短路阈值 + * @return 估算字节数 + */ + private long estimateValueBytes(Object value, + IdentityHashMap visited, + long limit) { + if (value == null) { + return 0L; + } + if (value instanceof CharSequence) { + return (long) ((CharSequence) value).length() * Character.BYTES; + } + if (value instanceof byte[]) { + return ((byte[]) value).length; + } + if (value instanceof Number || value instanceof Date) { + return 16L; + } + if (value instanceof Boolean || value instanceof Character) { + return 2L; + } + if (visited.put(value, Boolean.TRUE) != null) { + return 0L; + } + long bytes = 32L; + if (value instanceof Map) { + for (Object entryObject : ((Map) value).entrySet()) { + Map.Entry entry = (Map.Entry) entryObject; + bytes += estimateValueBytes(entry.getKey(), visited, limit - bytes); + bytes += estimateValueBytes(entry.getValue(), visited, limit - bytes); + if (bytes > limit) { + return bytes; + } + } + } else if (value instanceof Collection) { + for (Object item : (Collection) value) { + bytes += estimateValueBytes(item, visited, limit - bytes); + if (bytes > limit) { + return bytes; + } + } + } else if (value instanceof Iterable) { + // 任意 Iterable 可能是单次消费流,预算估算不得改变业务输入。 + bytes += 64L; + } else if (value.getClass().isArray()) { + int length = java.lang.reflect.Array.getLength(value); + for (int index = 0; index < length; index++) { + bytes += estimateValueBytes(java.lang.reflect.Array.get(value, index), visited, limit - bytes); + if (bytes > limit) { + return bytes; + } + } + } else { + bytes += 64L; + } + return bytes; + } + public void start(Map variables) { Trigger prev = TriggerContext.getCurrentTrigger(); @@ -214,38 +401,127 @@ public class Chain { // start 可能在 node 里执行一个新的 chain 的情况, // 需要清空父级 chain 的 Trigger TriggerContext.setCurrentTrigger(null); - updateStateSafely(state -> { - EnumSet fields = EnumSet.of(ChainStateField.STATUS); - state.setStatus(ChainStatus.RUNNING); - - if (variables != null && !variables.isEmpty()) { - state.getMemory().putAll(variables); - applyStartParameterAliases(state.getMemory(), variables); - fields.add(ChainStateField.MEMORY); - } - - if (StringUtil.noText(state.getChainDefinitionId())) { - state.setChainDefinitionId(definition.getId()); - fields.add(ChainStateField.CHAIN_DEFINITION_ID); - } - - return fields; - }); - - notifyEvent(new ChainStartEvent(this, variables)); - setStatusAndNotifyEvent(ChainStatus.RUNNING); - - // 调度入口节点 - List startNodes = definition.getStartNodes(); - for (Node startNode : startNodes) { - scheduleNode(startNode, null, TriggerType.START, 0); - } + initializeState(); + ensureStarted(variables); } finally { // 恢复父级 chain 的 Trigger TriggerContext.setCurrentTrigger(prev); } } + /** + * 恢复被入口触发器捕获的 READY 启动。 + * + *

入口意图先于 RUNNING 状态保存。若进程在两者之间退出,扫描器重新投递 + * START 触发器时由该方法补齐所有入口意图、恢复初始变量并推进到 RUNNING, + * 当前触发器随后继续执行,避免被误确认后永久停留在 READY。

+ * + * @param trigger 当前 START 触发器 + * @return 实例已经可以继续执行业务节点时为 {@code true} + */ + private boolean recoverStart(Trigger trigger) { + if (trigger == null || trigger.getType() != TriggerType.START) { + return false; + } + return ensureStarted(trigger.getStartVariables()); + } + + /** + * 幂等持久化入口意图并推进实例启动状态。 + * + * @param variables 首次启动变量;恢复时可来自首个稳定入口意图 + * @return 实例处于 RUNNING 时为 {@code true} + */ + private boolean ensureStarted(Map variables) { + AtomicBoolean active = new AtomicBoolean(); + executeWithLock(stateInstanceId, 10L, TimeUnit.SECONDS, () -> { + AtomicReference before = + new AtomicReference<>(); + AtomicBoolean started = new AtomicBoolean(); + ChainState current = chainStateRepository.load( + stateInstanceId); + if (current == null + || (current.getStatus() != ChainStatus.READY + && current.getStatus() != ChainStatus.RUNNING)) { + return null; + } + + List pendingStartNodes = new ArrayList<>(); + for (Node startNode : definition.getStartNodes()) { + NodeState nodeState = nodeStateRepository.load( + stateInstanceId, startNode.getId()); + if (!isCompletedStartNode(nodeState)) { + pendingStartNodes.add(startNode); + } + } + + List startTriggerIds = new ArrayList<>(); + if (current.getStatus() == ChainStatus.READY + && !pendingStartNodes.isEmpty()) { + // 首个稳定入口意图携带变量,确保该意图保存后可独立恢复输入。 + startTriggerIds.add(prepareStartNodeLocked( + pendingStartNodes.get(0), variables)); + } + + if (current.getStatus() == ChainStatus.READY) { + updateStateSafely(state -> { + before.set(state.getStatus()); + EnumSet fields = + EnumSet.noneOf(ChainStateField.class); + if (variables != null && !variables.isEmpty()) { + state.getMemory().putAll(variables); + applyStartParameterAliases( + state.getMemory(), variables); + fields.add(ChainStateField.MEMORY); + } + if (StringUtil.noText( + state.getChainDefinitionId())) { + state.setChainDefinitionId(definition.getId()); + fields.add( + ChainStateField.CHAIN_DEFINITION_ID); + } + return fields; + }); + } + + int preparedCount = startTriggerIds.isEmpty() ? 0 : 1; + for (int index = preparedCount; + index < pendingStartNodes.size(); + index++) { + startTriggerIds.add(prepareStartNodeLocked( + pendingStartNodes.get(index), null)); + } + + if (current.getStatus() == ChainStatus.READY) { + updateStateSafely(state -> { + EnumSet fields = + EnumSet.of(ChainStateField.STATUS); + state.setStatus(ChainStatus.RUNNING); + if (state.getStartedAt() <= 0) { + state.setStartedAt( + System.currentTimeMillis()); + fields.add(ChainStateField.STARTED_AT); + } + started.set(true); + return fields; + }); + } + active.set(true); + if (started.get()) { + notifyEvent(new ChainStartEvent(this, variables)); + notifyEvent(new ChainStatusChangeEvent( + this, ChainStatus.RUNNING, before.get())); + } + + // 状态提交后主动领取;崩溃时持久扫描仍会重新投递。 + for (String triggerId : startTriggerIds) { + getTriggerScheduler().fire(triggerId); + } + return null; + }); + return active.get(); + } + /** * 为开始节点输入参数补齐 `nodeId.paramName` 与 `paramName` 双向别名。 * 这样既兼容运行表单仅提交裸参数名,也兼容设计器内部统一保存完整引用路径。 @@ -291,12 +567,25 @@ public class Chain { public void executeNode(Node node, Trigger trigger) { try { EXECUTION_THREAD_LOCAL.set(this); - ChainState chainState = getState(); + assertTriggerClaimOwned(trigger); + ChainState chainState = loadFreshState(); + bindNodeExecutionState(chainState); + + if (chainState.getStatus() == ChainStatus.READY + && trigger.getType() == TriggerType.START) { + if (!recoverStart(trigger)) { + throw new IllegalStateException( + "Failed to recover READY workflow start: " + + stateInstanceId); + } + chainState = loadFreshState(); + bindNodeExecutionState(chainState); + } // 当前处于挂起状态 if (chainState.getStatus() == ChainStatus.SUSPEND) { updateStateSafely(state -> { - chainState.addSuspendNodeId(node.getId()); + state.addSuspendNodeId(node.getId()); return EnumSet.of(ChainStateField.SUSPEND_NODE_IDS); }); return; @@ -306,6 +595,15 @@ public class Chain { return; } + // 可恢复启动可能重放同一稳定入口触发器;已开始的入口节点直接确认。 + if (trigger.getType() == TriggerType.START) { + NodeState existing = nodeStateRepository.load( + stateInstanceId, node.getId()); + if (isCompletedStartNode(existing)) { + return; + } + } + String triggerEdgeId = trigger.getEdgeId(); if (shouldSkipNode(node, triggerEdgeId)) { return; @@ -313,44 +611,95 @@ public class Chain { Map nodeResult = null; Throwable error = null; + String executionAttemptKey = null; try { - NodeState nodeState = getNodeState(node.id); - - // 如果节点状态不是运行中,则更新为运行中 - // 目前只有 Loop 节点会处于 Running 状态,因为它会多次触发 - if (nodeState.getStatus() != NodeStatus.RUNNING) { - updateNodeStateSafely(node.id, s -> { - s.setStatus(NodeStatus.RUNNING); - s.recordExecute(triggerEdgeId); - return EnumSet.of(NodeStateField.EXECUTE_COUNT, NodeStateField.EXECUTE_EDGE_IDS, NodeStateField.STATUS); - }); - TriggerType type = trigger.getType(); - notifyEvent(new NodeStartEvent(this, node)); - } - // 只需记录执行次数 - else { - updateNodeStateSafely(node.id, s -> { - s.recordExecute(triggerEdgeId); - return EnumSet.of(NodeStateField.EXECUTE_COUNT, NodeStateField.EXECUTE_EDGE_IDS); - }); + AtomicBoolean nodeStarted = new AtomicBoolean(); + String candidateAttemptKey = + currentExecutionAttemptKey( + node.getId()); + // 首次创建、状态转换和执行计数在同一实例锁内完成,避免额外读取和首次执行空状态。 + NodeState activeNodeState = + updateNodeStateSafely(node.id, s -> { + nodeStarted.set(false); + EnumSet fields = EnumSet.of( + NodeStateField.EXECUTE_COUNT, + NodeStateField.EXECUTE_EDGE_IDS); + if (s.getStatus() != NodeStatus.RUNNING) { + s.setStatus(NodeStatus.RUNNING); + nodeStarted.set(true); + fields.add(NodeStateField.STATUS); + s.setExecutionAttemptKey( + candidateAttemptKey); + fields.add( + NodeStateField + .EXECUTION_ATTEMPT_KEY); + } + if (node.getCondition() == null) { + s.recordTrigger(triggerEdgeId); + fields.add(NodeStateField.TRIGGER_COUNT); + fields.add(NodeStateField.TRIGGER_EDGE_IDS); + } + s.recordExecute(triggerEdgeId); + return fields; + }); + executionAttemptKey = + activeNodeState + .getExecutionAttemptKey(); + if (nodeStarted.get()) { + notifyEvent(new NodeStartEvent( + this, + node, + executionAttemptKey, + activeNodeState.getStatus(), + getAuditInstanceId())); } - updateStateSafely(state -> { + ChainState nodeExecutionState = updateStateSafely(state -> { + long childExecutionCount = state.getChildExecutionCount() + 1; + executionBudget.checkChildExecutions(childExecutionCount); + state.setChildExecutionCount(childExecutionCount); state.addTriggerNodeId(node.id); - return EnumSet.of(ChainStateField.TRIGGER_NODE_IDS); + return EnumSet.of( + ChainStateField.TRIGGER_NODE_IDS, + ChainStateField.CHILD_EXECUTION_COUNT); }); + bindNodeExecutionState(nodeExecutionState); + executionBudget.checkDuration(chainState.getStartedAt(), System.currentTimeMillis()); nodeResult = node.execute(this); + assertTriggerClaimOwned(trigger); + } catch (TriggerClaimLostException | RetryableTriggerException claimLost) { + throw claimLost; } catch (Throwable throwable) { log.error("Node execute error", throwable); error = throwable; } - handleNodeResult(node, nodeResult, triggerEdgeId, error); + // 结果提交入口会在实例锁内重读状态,统一拦截取消、超时和其他终态。 + handleNodeResult( + node, + nodeResult, + triggerEdgeId, + error, + executionAttemptKey); } finally { + executionStateSnapshot = null; + executionStateSnapshotLock = null; + NODE_EXECUTION_STATE_VIEW.remove(); EXECUTION_THREAD_LOCAL.remove(); } } + /** + * 验证当前触发器仍由本工作线程持有,阻止失去租约的工作线程提交状态。 + * + * @param trigger 当前触发器 + */ + private void assertTriggerClaimOwned(Trigger trigger) { + if (triggerScheduler != null) { + triggerScheduler.assertClaimOwned(trigger); + } + } + public NodeState getNodeState(String nodeId) { return getNodeState(this.stateInstanceId, nodeId); } @@ -360,31 +709,96 @@ public class Chain { } public T executeWithLock(String instanceId, long timeout, TimeUnit unit, Supplier action) { - try (ChainLock lock = chainStateRepository.getLock(instanceId, timeout, unit)) { - if (!lock.isAcquired()) { - throw new ChainLockTimeoutException("Failed to acquire lock for instance: " + instanceId); - } + Map heldLocks = HELD_INSTANCE_LOCKS.get(); + if (heldLocks != null && heldLocks.containsKey(instanceId)) { + assertInstanceLockOwned(instanceId); return action.get(); } + if (heldLocks == null) { + heldLocks = new HashMap<>(); + HELD_INSTANCE_LOCKS.set(heldLocks); + } + Map lockScope = heldLocks; + try { + try (ChainLock lock = chainStateRepository.getLock(instanceId, timeout, unit)) { + if (!lock.isAcquired()) { + throw new ChainLockTimeoutException("Failed to acquire lock for instance: " + instanceId); + } + lockScope.put(instanceId, lock); + try { + T result = action.get(); + assertInstanceLockOwned(instanceId); + return result; + } finally { + lockScope.remove(instanceId); + } + } + } finally { + if (lockScope.isEmpty()) { + HELD_INSTANCE_LOCKS.remove(); + } + } + } + + /** + * 验证当前线程持有的实例锁仍然有效。 + * + * @param instanceId 工作流实例 ID + */ + private void assertInstanceLockOwned(String instanceId) { + ChainLock lock = currentInstanceLock(instanceId); + if (lock == null || !lock.isValid()) { + throw new ChainLockTimeoutException( + "Workflow instance lock ownership lost: " + instanceId); + } + } + + /** + * 获取当前线程持有的实例锁。 + * + * @param instanceId 工作流实例 ID + * @return 当前实例锁;未持有时为 {@code null} + */ + private ChainLock currentInstanceLock( + String instanceId) { + Map heldLocks = + HELD_INSTANCE_LOCKS.get(); + return heldLocks == null + ? null + : heldLocks.get(instanceId); + } + + /** + * 显式创建当前工作流实例状态。 + * + * @return 已存在或新创建的状态 + */ + public ChainState initializeState() { + ChainState state = chainStateRepository.create(stateInstanceId); + if (state == null) { + throw new IllegalStateException("Unable to initialize chain state: " + stateInstanceId); + } + return state; } private boolean shouldSkipNode(Node node, String edgeId) { + NodeCondition condition = node.getCondition(); + if (condition == null) { + return false; + } return executeWithLock(stateInstanceId, 10, TimeUnit.SECONDS, () -> { NodeState newState = updateNodeStateSafely(node.id, s -> { s.recordTrigger(edgeId); return EnumSet.of(NodeStateField.TRIGGER_COUNT, NodeStateField.TRIGGER_EDGE_IDS); }); - NodeCondition condition = node.getCondition(); - if (condition == null) { - return false; - } Map prevResult = Collections.emptyMap(); boolean shouldSkipNode = !condition.check(this, newState, prevResult); if (shouldSkipNode) { updateStateSafely(state -> { - state.addUncheckedNodeId(node.id); - return EnumSet.of(ChainStateField.UNCHECKED_NODE_IDS); + return state.addUncheckedNodeId(node.id) + ? EnumSet.of(ChainStateField.UNCHECKED_NODE_IDS) + : null; }); } else { updateStateSafely(state -> { @@ -400,7 +814,53 @@ public class Chain { } - private void handleNodeResult(Node node, Map prevNodeResult, String triggerEdgeId, Throwable error) { + private void handleNodeResult(Node node, + Map prevNodeResult, + String triggerEdgeId, + Throwable error, + String executionAttemptKey) { + List deferredActions = new ArrayList<>(); + executeWithLock(stateInstanceId, 10, TimeUnit.SECONDS, () -> { + assertTriggerClaimOwned(TriggerContext.getCurrentTrigger()); + ChainState latestState = loadFreshState(); + cacheExecutionState( + stateInstanceId, latestState); + if (latestState == null) { + throw new IllegalStateException("Chain state not found: " + stateInstanceId); + } + if (latestState.getStatus() != ChainStatus.RUNNING) { + return null; + } + DEFERRED_EVENT_ACTIONS.set(deferredActions); + try { + handleNodeResultLocked( + node, + prevNodeResult, + triggerEdgeId, + error, + executionAttemptKey); + } finally { + DEFERRED_EVENT_ACTIONS.remove(); + } + return null; + }); + deferredActions.forEach(Runnable::run); + } + + /** + * 在实例锁保护下合并节点结果并调度后继节点。 + * + * @param node 已执行节点 + * @param prevNodeResult 节点输出 + * @param triggerEdgeId 触发边 ID + * @param error 节点执行异常 + * @param executionAttemptKey 进入节点时捕获的业务尝试键 + */ + private void handleNodeResultLocked(Node node, + Map prevNodeResult, + String triggerEdgeId, + Throwable error, + String executionAttemptKey) { ChainStatus finalChainStatus = null; NodeStatus finalNodeStatus = null; try { @@ -470,7 +930,7 @@ public class Chain { return EnumSet.of(NodeStateField.ERROR, NodeStateField.STATUS); }); - eventManager.notifyNodeError(error, node, prevNodeResult, this); + deferOrRun(() -> eventManager.notifyNodeError(error, node, prevNodeResult, this)); if (node.isRetryEnable() && node.getMaxRetryCount() > 0 @@ -495,7 +955,13 @@ public class Chain { state.setStatus(nodeStatus); return EnumSet.of(NodeStateField.STATUS); }); - notifyEvent(new NodeEndEvent(this, node, prevNodeResult, error)); + notifyEvent(new NodeEndEvent( + this, + node, + prevNodeResult, + error, + nodeStatus, + executionAttemptKey)); } if (finalChainStatus != null) { @@ -503,7 +969,7 @@ public class Chain { // chain 执行结束 if (finalChainStatus.isTerminal()) { - eventManager.notifyEvent(new ChainEndEvent(this), this); + notifyEvent(new ChainEndEvent(this)); // 执行结束,但是未执行成功,失败和取消等 // 更新父级链的状态 @@ -539,28 +1005,44 @@ public class Chain { } NodeState nodeState = getNodeState(node.getId()); - // 如果达到最大循环次数限制,则调度向外的节点 - if (node.getMaxLoopCount() > 0 && nodeState.getLoopCount() >= node.getMaxLoopCount()) { - scheduleOutwardNodes(node, result); - return; - } + int completedLoopCount = Math.addExact(nodeState.getLoopCount(), 1); + executionBudget.checkIterations(node.getId(), completedLoopCount); // 检查循环中断条件,如果满足则调度向外的节点 NodeCondition breakCondition = node.getLoopBreakCondition(); - if (breakCondition != null && breakCondition.check(this, nodeState, result)) { + boolean shouldBreak = breakCondition != null + && breakCondition.check(this, nodeState, result); + if (shouldBreak || completedLoopCount >= node.getMaxLoopCount()) { + resetNodeLoopCount(node.getId()); scheduleOutwardNodes(node, result); return; } - // 增加循环计数并重新调度当前节点 + // 记录已经完成的执行次数;下一次执行的零基索引与该值一致。 updateNodeStateSafely(node.getId(), s -> { - s.setLoopCount(s.getLoopCount() + 1); + s.setLoopCount(completedLoopCount); return EnumSet.of(NodeStateField.LOOP_COUNT); }); scheduleNode(node, byEdigeId, TriggerType.LOOP, node.getLoopIntervalMs()); } + /** + * 清理一次节点循环生命周期的计数,保证节点被外层循环再次触发时从零开始。 + * + * @param nodeId 节点 ID + */ + private void resetNodeLoopCount(String nodeId) { + NodeState currentState = getNodeState(nodeId); + if (currentState.getLoopCount() == 0) { + return; + } + updateNodeStateSafely(nodeId, state -> { + state.setLoopCount(0); + return EnumSet.of(NodeStateField.LOOP_COUNT); + }); + } + private void scheduleOutwardNodes(Node node, Map result) { List edges = definition.getOutwardEdge(node.getId()); @@ -608,10 +1090,11 @@ public class Chain { scheduleSuccess = true; } else { updateStateSafely(state -> { - state.addUncheckedEdgeId(edge.getId()); - return EnumSet.of(ChainStateField.UNCHECKED_EDGE_IDS); + return state.addUncheckedEdgeId(edge.getId()) + ? EnumSet.of(ChainStateField.UNCHECKED_EDGE_IDS) + : null; }); - eventManager.notifyEvent(new EdgeConditionCheckFailedEvent(this, edge, node, result), this); + notifyEvent(new EdgeConditionCheckFailedEvent(this, edge, node, result)); } } @@ -643,12 +1126,164 @@ public class Chain { public void scheduleNode(Node node, String edgeId, TriggerType type, long delayMs) { + scheduleNode(node, edgeId, type, delayMs, null, null); + } + + /** + * 调度循环体直属分支并写入稳定的代际游标。 + * + * @param node 目标节点 + * @param edgeId 触发边 ID + * @param delayMs 延迟毫秒数 + * @param loopNodeId 循环节点 ID + * @param cursor 本轮分支游标 + */ + public void scheduleLoopChild(Node node, + String edgeId, + long delayMs, + String loopNodeId, + Trigger.LoopCursor cursor) { + scheduleNode(node, edgeId, TriggerType.CHILD, delayMs, loopNodeId, cursor); + } + + /** + * 创建并持久化节点触发器。 + * + * @param node 目标节点 + * @param edgeId 触发边 ID + * @param type 触发类型 + * @param delayMs 延迟毫秒数 + * @param loopNodeId 需要覆盖的循环节点 ID + * @param cursor 循环代际游标 + */ + private void scheduleNode(Node node, + String edgeId, + TriggerType type, + long delayMs, + String loopNodeId, + Trigger.LoopCursor cursor) { + executeWithLock(stateInstanceId, 10L, TimeUnit.SECONDS, () -> { + scheduleNodeLocked(node, edgeId, type, delayMs, loopNodeId, cursor); + return null; + }); + } + + /** + * 在实例锁保护下创建并持久化节点触发器。 + * + * @param node 目标节点 + * @param edgeId 触发边 ID + * @param type 触发类型 + * @param delayMs 延迟毫秒数 + * @param loopNodeId 需要覆盖的循环节点 ID + * @param cursor 循环代际游标 + */ + private void scheduleNodeLocked(Node node, + String edgeId, + TriggerType type, + long delayMs, + String loopNodeId, + Trigger.LoopCursor cursor) { + scheduleNodeLocked( + node, edgeId, type, delayMs, loopNodeId, cursor, + null, false, true, null); + } + + /** + * 在状态切换前持久化可重放的入口触发器意图。 + * + * @param node 入口节点 + * @param startVariables 首个入口意图携带的启动变量 + * @return 稳定触发器 ID + */ + private String prepareStartNodeLocked( + Node node, Map startVariables) { + String stableId = "start-" + + UUID.nameUUIDFromBytes( + (stateInstanceId + '\n' + node.getId()) + .getBytes(java.nio.charset.StandardCharsets.UTF_8)); + scheduleNodeLocked( + node, null, TriggerType.START, 1_000L, null, null, + stableId, true, false, startVariables); + return stableId; + } + + /** + * 判断稳定入口触发器是否已经完成状态提交。 + * + *

RUNNING 不能视为完成:进程可能在执行计数递增后、业务结果提交前崩溃, + * 此时必须允许同一稳定触发器重放。

+ * + * @param nodeState 入口节点状态 + * @return 已提交终态时为 {@code true} + */ + private boolean isCompletedStartNode(NodeState nodeState) { + if (nodeState == null || nodeState.getStatus() == null) { + return false; + } + return nodeState.getStatus() == NodeStatus.SUCCEEDED + || nodeState.getStatus() == NodeStatus.SUSPEND + || nodeState.getStatus() == NodeStatus.ERROR + || nodeState.getStatus() == NodeStatus.FAILED; + } + + /** + * 在实例锁保护下创建并持久化节点触发器。 + * + * @param node 目标节点 + * @param edgeId 触发边 ID + * @param type 触发类型 + * @param delayMs 延迟毫秒数 + * @param loopNodeId 循环节点 ID + * @param cursor 循环游标 + * @param stableTriggerId 可选稳定触发器 ID + * @param onlyIfAbsent 是否仅在仓储中不存在时保存 + * @param requireActive 是否要求实例已处于运行态 + * @param startVariables 首个入口意图携带的启动变量 + */ + private void scheduleNodeLocked(Node node, + String edgeId, + TriggerType type, + long delayMs, + String loopNodeId, + Trigger.LoopCursor cursor, + String stableTriggerId, + boolean onlyIfAbsent, + boolean requireActive, + Map startVariables) { + if (requireActive && !isExecutionActiveLocked()) { + return; + } Trigger trigger = new Trigger(); + trigger.setId(stableTriggerId); trigger.setStateInstanceId(stateInstanceId); trigger.setEdgeId(edgeId); trigger.setNodeId(node.getId()); trigger.setType(type); + trigger.setExecutionLane(executionLane); + trigger.setStartVariables(startVariables); trigger.setTriggerAt(System.currentTimeMillis() + delayMs); + Trigger currentTrigger = TriggerContext.getCurrentTrigger(); + String fencingClaimId = currentFencingClaimId(); + long claimGeneration = currentClaimGeneration(); + long lockFencingToken = currentLockFencingToken(stateInstanceId); + trigger.setRequiredLockFencingToken(lockFencingToken); + if (claimGeneration > 0L && StringUtil.hasText(fencingClaimId)) { + trigger.setRequiredFencingClaimId(fencingClaimId); + trigger.setRequiredFencingToken(claimGeneration); + } + if (type == TriggerType.RETRY && currentTrigger != null) { + String logicalExecutionId = StringUtil.hasText(currentTrigger.getLogicalExecutionId()) + ? currentTrigger.getLogicalExecutionId() + : currentTrigger.getId(); + trigger.setLogicalExecutionId(logicalExecutionId); + } + if (currentTrigger != null && !currentTrigger.getLoopCursors().isEmpty()) { + trigger.setLoopCursors(new LinkedHashMap<>(currentTrigger.getLoopCursors())); + } + if (loopNodeId != null && cursor != null) { + trigger.getLoopCursors().put(loopNodeId, cursor); + } if (edgeId != null) { updateStateSafely(state -> { @@ -656,10 +1291,386 @@ public class Chain { return EnumSet.of(ChainStateField.TRIGGER_EDGE_IDS); }); - eventManager.notifyEvent(new EdgeTriggerEvent(this, trigger), this); + notifyEvent(new EdgeTriggerEvent(this, trigger)); } - getTriggerScheduler().schedule(trigger); + if (onlyIfAbsent) { + getTriggerScheduler().scheduleIfAbsent(trigger); + } else { + getTriggerScheduler().schedule(trigger); + } + } + + /** + * 在已持有实例锁时判断工作流是否仍可推进。 + * + *

优先复用当前触发器内已加载或成功提交的状态快照。实例锁保证此期间其他执行者 + * 无法提交取消、超时或终态,因此无需再次访问状态仓储。

+ * + * @return 当前实例处于运行态时返回 {@code true} + */ + private boolean isExecutionActiveLocked() { + assertInstanceLockOwned(stateInstanceId); + ChainState state = reusableExecutionState(stateInstanceId); + if (state == null) { + state = loadFreshState(); + cacheExecutionState(stateInstanceId, state); + } + return state != null && state.getStatus() == ChainStatus.RUNNING; + } + + /** + * 获取当前触发器认领代际。 + * + * @param instanceId 工作流实例 ID + * @return 当前认领代际;非触发器调用为 {@code 0} + */ + private long currentClaimGeneration(String instanceId) { + Trigger trigger = TriggerContext.getCurrentTrigger(); + return trigger != null && Objects.equals(instanceId, trigger.getStateInstanceId()) + ? trigger.getFencingToken() + : 0L; + } + + /** + * 获取当前触发器认领代际。 + * + * @return 当前认领代际;非触发器调用为 {@code 0} + */ + public long currentClaimGeneration() { + return currentClaimGeneration(stateInstanceId); + } + + /** + * 获取当前线程持有的实例锁 fencing token。 + * + * @param instanceId 工作流实例 ID + * @return 当前锁 token;本地锁或未持锁时为 {@code 0} + */ + private long currentLockFencingToken(String instanceId) { + Map heldLocks = HELD_INSTANCE_LOCKS.get(); + ChainLock lock = heldLocks == null ? null : heldLocks.get(instanceId); + return lock == null ? 0L : lock.getFencingToken(); + } + + /** + * 获取当前线程持有的本实例锁 fencing token。 + * + * @return 当前锁 token;本地锁或未持锁时为 {@code 0} + */ + public long currentInstanceLockFencingToken() { + return currentLockFencingToken(stateInstanceId); + } + + /** + * 获取当前触发器认领 ID。 + * + * @param instanceId 工作流实例 ID + * @return 当前触发器 ID;非触发器调用为 {@code null} + */ + private String currentFencingClaimId(String instanceId) { + Trigger trigger = TriggerContext.getCurrentTrigger(); + return trigger != null && Objects.equals(instanceId, trigger.getStateInstanceId()) + ? trigger.getId() + : null; + } + + /** + * 获取当前触发器认领 ID,供循环结果等派生状态原子提交使用。 + * + * @return 当前触发器 ID;非触发器调用为 {@code null} + */ + public String currentFencingClaimId() { + return currentFencingClaimId(stateInstanceId); + } + + /** + * 在实例锁保护下原子追加循环结果。 + * + * @param resultId 循环结果 ID + * @param iterationIndex 迭代序号 + * @param outputValues 本轮输出 + */ + public void appendLoopResult( + String resultId, int iterationIndex, Map outputValues) { + executeWithLock(stateInstanceId, 10L, TimeUnit.SECONDS, () -> { + loopResultRepository.append( + stateInstanceId, + currentLockFencingToken(stateInstanceId), + currentFencingClaimId(stateInstanceId), + currentClaimGeneration(stateInstanceId), + resultId, + iterationIndex, + outputValues); + return null; + }); + } + + /** + * 使用当前实例锁和触发器认领守卫物化循环输入。 + * + * @param resultId 循环结果 ID + * @param items 原始输入 + * @param maxItems 最大元素数 + * @return 已物化元素数量 + */ + public int storeLoopInput( + String resultId, Iterable items, long maxItems) { + return loopResultRepository.storeInput( + stateInstanceId, + currentLockFencingToken(stateInstanceId), + currentFencingClaimId(stateInstanceId), + currentClaimGeneration(stateInstanceId), + resultId, + items, + maxItems); + } + + /** + * 使用稳定的触发器认领守卫在实例锁外物化循环输入。 + * + *

实例锁是短临界区资源,其他合法分支提交会推进其 fencing token, + * 因而锁外长 I/O 仅绑定触发器 claim。发布结果时会重新获取实例锁并校验 + * resultId 与物化代际。

+ * + * @param resultId 循环结果 ID + * @param items 原始输入 + * @param maxItems 最大元素数 + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @return 已物化元素数量 + */ + public int storeLoopInputOutsideLock( + String resultId, + Iterable items, + long maxItems, + String claimId, + long claimGeneration) { + return loopResultRepository.storeInput( + stateInstanceId, + 0L, + claimId, + claimGeneration, + resultId, + items, + maxItems); + } + + /** + * 使用稳定触发器认领守卫,在实例锁外接收并分块保存推送式输入。 + * + * @param resultId 循环结果 ID + * @param producer 输入生产者 + * @param maxItems 最大元素数 + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @return 已物化元素数量 + */ + public int storeProducedLoopInputOutsideLock( + String resultId, + LoopResultRepository.InputProducer producer, + long maxItems, + String claimId, + long claimGeneration) { + return loopResultRepository.storeProducedInput( + stateInstanceId, + 0L, + claimId, + claimGeneration, + resultId, + producer, + maxItems); + } + + /** + * 使用当前实例锁和触发器认领守卫清理循环输入。 + * + * @param resultId 循环结果 ID + */ + public void removeLoopInput(String resultId) { + loopResultRepository.removeInput( + stateInstanceId, + currentLockFencingToken(stateInstanceId), + currentFencingClaimId(stateInstanceId), + currentClaimGeneration(stateInstanceId), + resultId); + } + + /** + * 透明还原循环累计结果引用。 + * + * @param value 可能包含循环引用的值 + * @return 业务可见值 + */ + public Object resolveResultReferences(Object value) { + return loopResultRepository.resolveReferences(value); + } + + /** + * 判断当前工作流实例是否仍允许执行和推进。 + * + * @return 状态存在且为运行中时返回 {@code true} + */ + private boolean isExecutionActive() { + // 取消、超时等状态可能由其他线程提交,此处必须绕过执行期快照。 + ChainState state = loadFreshState(); + cacheExecutionState(stateInstanceId, state); + return state != null && state.getStatus() == ChainStatus.RUNNING; + } + + /** + * 从仓储读取最新状态,判断当前实例是否仍允许推进。 + * + * @return 最新状态为运行中时返回 {@code true} + */ + public boolean isExecutionActiveNow() { + return isExecutionActive(); + } + + /** + * 设置本实例派生触发器使用的执行通道。 + * + * @param executionLane 执行通道;{@code null} 表示默认通道 + */ + public void setExecutionLane(String executionLane) { + this.executionLane = executionLane; + } + + /** + * 设置由父子工作流调用链贡献的基础嵌套深度。 + * + * @param nestedDepthBase 非负基础深度 + */ + public void setNestedDepthBase(int nestedDepthBase) { + this.nestedDepthBase = Math.max(0, nestedDepthBase); + } + + /** + * 获取父子工作流调用链贡献的基础嵌套深度。 + * + * @return 非负基础深度 + */ + public int getNestedDepthBase() { + return nestedDepthBase; + } + + /** + * 生成当前节点副作用操作的稳定幂等键。 + * + *

同一持久化触发器发生租约恢复或重复投递时返回相同键;直接执行单节点且没有触发器 + * 上下文时返回 {@code null},保持原有每次调用均执行的语义。

+ * + * @param nodeId 当前节点 ID + * @return 稳定幂等键;无持久化触发器上下文时返回 {@code null} + */ + public String currentExecutionIdempotencyKey(String nodeId) { + Trigger trigger = TriggerContext.getCurrentTrigger(); + if (trigger == null || StringUtil.noText(trigger.getId())) { + return null; + } + String logicalExecutionId = StringUtil.hasText(trigger.getLogicalExecutionId()) + ? trigger.getLogicalExecutionId() + : trigger.getId(); + return stateInstanceId + ":" + nodeId + ":" + logicalExecutionId; + } + + /** + * 生成当前节点本次业务尝试的稳定键。 + * + *

基础设施重新投递同一触发器时键保持不变,业务重试生成新触发器时键随之变化, + * 可用于执行步骤等“每次业务尝试一条”的幂等记录。

+ * + * @param nodeId 当前节点 ID + * @return 当前业务尝试稳定键;无持久化触发器上下文时返回 {@code null} + */ + public String currentExecutionAttemptKey(String nodeId) { + Trigger trigger = TriggerContext.getCurrentTrigger(); + if (trigger == null || StringUtil.noText(trigger.getId())) { + return null; + } + return stateInstanceId + ":" + nodeId + ":" + trigger.getId(); + } + + /** + * 将当前工作流实例标记为取消。 + * + * @param message 取消原因 + * @return 本次是否将非终态实例转换为取消状态 + */ + public boolean cancel(String message) { + AtomicReference changed = new AtomicReference<>(false); + AtomicReference before = new AtomicReference<>(); + updateStateSafely(state -> { + if (state.getStatus() != null && state.getStatus().isTerminal()) { + return null; + } + before.set(state.getStatus()); + state.setStatus(ChainStatus.CANCELLED); + state.setMessage(message); + changed.set(true); + return EnumSet.of(ChainStateField.STATUS, ChainStateField.MESSAGE); + }); + if (changed.get()) { + notifyEvent(new ChainStatusChangeEvent( + this, ChainStatus.CANCELLED, before.get())); + } + return changed.get(); + } + + /** + * 将不可继续投递的实例幂等收敛为失败终态并发布统一终态事件。 + * + *

该入口用于触发器耗尽或不可恢复错误。状态、错误和消息先在实例锁内提交, + * 再发布状态、错误和结束事件,使同步等待、审计记录和定义快照清理观察到同一终态。

+ * + * @param cause 最终失败原因 + * @return 本次完成终态转换或实例此前已终止时为 {@code true} + */ + public boolean failTerminal(Throwable cause) { + Throwable failure = cause == null + ? new ChainException("Workflow execution failed") + : cause; + AtomicBoolean changed = new AtomicBoolean(); + AtomicReference before = + new AtomicReference<>(); + List deferredActions = new ArrayList<>(); + executeWithLock(stateInstanceId, 10L, TimeUnit.SECONDS, () -> { + updateStateSafely(state -> { + if (state.getStatus() != null + && state.getStatus().isTerminal()) { + return null; + } + before.set(state.getStatus()); + state.setStatus(ChainStatus.FAILED); + state.setError(new ExceptionSummary(failure)); + state.setMessage(failure.getMessage()); + changed.set(true); + return EnumSet.of( + ChainStateField.STATUS, + ChainStateField.ERROR, + ChainStateField.MESSAGE); + }); + if (changed.get()) { + DEFERRED_EVENT_ACTIONS.set(deferredActions); + try { + notifyEvent(new ChainStatusChangeEvent( + this, ChainStatus.FAILED, before.get())); + deferOrRun(() -> + eventManager.notifyChainError(failure, this)); + notifyEvent(new ChainEndEvent(this)); + } finally { + DEFERRED_EVENT_ACTIONS.remove(); + } + } + return null; + }); + deferredActions.forEach(Runnable::run); + ChainState state = chainStateRepository.load( + stateInstanceId); + return changed.get() + || (state != null + && state.getStatus() != null + && state.getStatus().isTerminal()); } @@ -676,10 +1687,24 @@ public class Chain { }); setStatusAndNotifyEvent(ChainStatus.FAILED); - eventManager.notifyChainError(throwable, this); + deferOrRun(() -> eventManager.notifyChainError(throwable, this)); return ChainStatus.FAILED; } + /** + * 在实例锁内仅记录事件动作,离开锁后按原顺序执行监听器 I/O。 + * + * @param action 事件动作 + */ + private void deferOrRun(Runnable action) { + List deferredActions = DEFERRED_EVENT_ACTIONS.get(); + if (deferredActions == null) { + action.run(); + } else { + deferredActions.add(action); + } + } + public void suspend() { setStatusAndNotifyEvent(ChainStatus.SUSPEND); } @@ -773,19 +1798,236 @@ public class Chain { this.nodeStateRepository = nodeStateRepository; } + /** + * 获取循环累计结果仓储。 + * + * @return 循环累计结果仓储 + */ + public LoopResultRepository getLoopResultRepository() { + return loopResultRepository; + } + + /** + * 设置循环累计结果仓储。 + * + * @param loopResultRepository 循环累计结果仓储;为空时使用进程内实现 + */ + public void setLoopResultRepository(LoopResultRepository loopResultRepository) { + this.loopResultRepository = loopResultRepository == null + ? new InMemoryLoopResultRepository() + : loopResultRepository; + } + public String getStateInstanceId() { return stateInstanceId; } + /** + * 获取当前工作流状态。 + * + * @return 仓储中的最新工作流状态 + */ public ChainState getState() { + return loadFreshState(); + } + + /** + * 获取当前节点执行入口捕获的状态视图。 + * + *

节点业务逻辑和参数解析应使用该方法,保证一次节点执行内引用同一个 + * point-in-time 状态并消除重复仓储 I/O。若当前线程不在节点执行上下文中, + * 则退化为读取仓储最新状态,不改变外部调用语义。

+ * + * @return 当前节点执行状态视图,或仓储中的最新状态 + */ + public ChainState getExecutionState() { + NodeExecutionStateView view = + NODE_EXECUTION_STATE_VIEW.get(); + return view != null && view.chain == this + ? view.state + : loadFreshState(); + } + + /** + * 获取节点审计归属的顶级执行实例 ID。 + * + *

新状态直接读取持久字段。兼容升级前仅保存 + * {@link ChainState#getParentInstanceId()} 的状态时,首次按父链解析并回填, + * 后续节点无需再次递归读取祖先状态。

+ * + * @return 顶级审计实例 ID + * @throws IllegalStateException 状态缺失、父链断裂或形成环时抛出 + */ + public String getAuditInstanceId() { + ChainState current = + getExecutionState(); + if (current == null) { + throw new IllegalStateException( + "Chain state not found: " + + stateInstanceId); + } + if (StringUtil.hasText( + current.getAuditInstanceId())) { + return current.getAuditInstanceId(); + } + + Set visited = + new HashSet<>(); + visited.add(current.getInstanceId()); + String resolvedInstanceId = + current.getInstanceId(); + while (StringUtil.hasText( + current.getParentInstanceId())) { + String parentInstanceId = + current.getParentInstanceId(); + if (!visited.add(parentInstanceId)) { + throw new IllegalStateException( + "Workflow parent state cycle detected: " + + parentInstanceId); + } + current = chainStateRepository.load( + parentInstanceId); + if (current == null) { + throw new IllegalStateException( + "Workflow parent state not found: " + + parentInstanceId); + } + if (StringUtil.hasText( + current.getAuditInstanceId())) { + resolvedInstanceId = + current.getAuditInstanceId(); + break; + } + resolvedInstanceId = + current.getInstanceId(); + } + + String rootInstanceId = + resolvedInstanceId; + ChainState updated = updateStateSafely(state -> { + if (StringUtil.hasText( + state.getAuditInstanceId())) { + return null; + } + state.setAuditInstanceId( + rootInstanceId); + return EnumSet.of( + ChainStateField.AUDIT_INSTANCE_ID); + }); + return updated.getAuditInstanceId(); + } + + /** + * 获取指定实例的最新工作流状态。 + * + * @param stateInstanceId 工作流实例 ID + * @return 指定实例的最新状态 + */ + public ChainState getState(String stateInstanceId) { return chainStateRepository.load(stateInstanceId); } - public ChainState getState(String stateInstanceId) { + /** + * 从仓储读取当前实例的最新状态。 + * + * @return 当前实例的最新状态 + */ + private ChainState loadFreshState() { return chainStateRepository.load(stateInstanceId); } + /** + * 在当前触发器执行线程内更新可复用状态快照。 + * + * @param instanceId 状态实例 ID + * @param state 最新状态 + */ + private void cacheExecutionState(String instanceId, ChainState state) { + if (EXECUTION_THREAD_LOCAL.get() == this + && Objects.equals(stateInstanceId, instanceId)) { + executionStateSnapshot = state; + executionStateSnapshotLock = + currentInstanceLock(instanceId); + NodeExecutionStateView view = + NODE_EXECUTION_STATE_VIEW.get(); + if (view != null && view.chain == this) { + view.state = state; + } + } + } + + /** + * 绑定当前线程单次节点执行所使用的状态视图。 + * + * @param state 节点入口的状态快照 + */ + private void bindNodeExecutionState(ChainState state) { + NODE_EXECUTION_STATE_VIEW.set( + new NodeExecutionStateView(this, state)); + } + + /** + * 单次节点执行的线程隔离状态视图。 + */ + private static final class NodeExecutionStateView { + private final Chain chain; + private ChainState state; + + /** + * 创建节点执行状态视图。 + * + * @param chain 所属执行链 + * @param state 节点入口状态 + */ + private NodeExecutionStateView( + Chain chain, ChainState state) { + this.chain = chain; + this.state = state; + } + } + + /** + * 获取当前执行线程可安全尝试提交的状态快照。 + * + *

仅复用由当前同一把实例锁读取或提交的快照。锁外读取和上一临界区快照均返回 + * {@code null},避免并发取消、超时或变量更新被旧状态遮蔽。

+ * + * @param instanceId 状态实例 ID + * @return 当前执行快照;不可复用时为 {@code null} + */ + private ChainState reusableExecutionState(String instanceId) { + ChainLock currentLock = + currentInstanceLock(instanceId); + return EXECUTION_THREAD_LOCAL.get() == this + && Objects.equals( + stateInstanceId, instanceId) + && currentLock != null + && currentLock.isValid() + && currentLock + == executionStateSnapshotLock + ? executionStateSnapshot + : null; + } + public void setStateInstanceId(String stateInstanceId) { this.stateInstanceId = stateInstanceId; } + + /** + * 获取当前实例的执行预算。 + * + * @return 执行预算 + */ + public ExecutionBudget getExecutionBudget() { + return executionBudget; + } + + /** + * 设置当前实例的执行预算。 + * + * @param executionBudget 执行预算;为空时恢复宽松缺省值 + */ + public void setExecutionBudget(ExecutionBudget executionBudget) { + this.executionBudget = executionBudget == null ? ExecutionBudget.defaults() : executionBudget; + } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainDefinition.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainDefinition.java index a791cf0..4c94a73 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainDefinition.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainDefinition.java @@ -20,16 +20,25 @@ import com.easyagents.flow.core.util.StringUtil; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.UUID; public class ChainDefinition implements Serializable { + private static final long serialVersionUID = -3183115191738959423L; + protected String id; protected String name; protected String description; protected List nodes; protected List edges; + /** + * 由节点和边派生的只读图索引,不参与序列化。 + */ + private transient volatile GraphIndex graphIndex; public ChainDefinition() { } @@ -64,6 +73,7 @@ public class ChainDefinition implements Serializable { public void setNodes(List nodes) { this.nodes = nodes; + invalidateGraphIndex(); } public List getEdges() { @@ -72,27 +82,45 @@ public class ChainDefinition implements Serializable { public void setEdges(List edges) { this.edges = edges; + invalidateGraphIndex(); } + /** + * 获取指定节点的全部出边。 + * + * @param nodeId 节点 ID + * @return 保持定义顺序的出边副本 + */ public List getOutwardEdge(String nodeId) { - List result = new ArrayList<>(); - for (Edge edge : edges) { - if (nodeId.equals(edge.getSource())) { - result.add(edge); - } - } - return result; + List outwardEdges = graphIndex().outwardEdgesByNode.get(nodeId); + return outwardEdges == null ? Collections.emptyList() : new ArrayList<>(outwardEdges); } + /** + * 获取指定节点的全部入边。 + * + * @param nodeId 节点 ID + * @return 保持定义顺序的入边副本 + */ public List getInwardEdge(String nodeId) { - List result = new ArrayList<>(); - for (Edge edge : edges) { - if (nodeId.equals(edge.getTarget())) { - result.add(edge); - } - } - return result; + List inwardEdges = graphIndex().inwardEdgesByNode.get(nodeId); + return inwardEdges == null ? Collections.emptyList() : new ArrayList<>(inwardEdges); + } + + /** + * 获取循环节点已编译的直属分支调度描述。 + * + * @param loopNodeId 循环节点 ID + * @return 保持定义顺序的不可变调度描述 + */ + public List getLoopChildDispatches( + String loopNodeId) { + List dispatches = + graphIndex().loopChildrenByNode.get(loopNodeId); + return dispatches == null + ? Collections.emptyList() + : dispatches; } public void addNode(Node node) { @@ -105,31 +133,21 @@ public class ChainDefinition implements Serializable { } nodes.add(node); - -// if (this.edges != null) { -// for (Edge edge : edges) { -// if (node.getId().equals(edge.getSource())) { -// node.addOutwardEdge(edge); -// } else if (node.getId().equals(edge.getTarget())) { -// node.addInwardEdge(edge); -// } -// } -// } + invalidateGraphIndex(); } + /** + * 按 ID 获取节点。 + * + * @param id 节点 ID + * @return 对应节点,不存在时返回 {@code null} + */ public Node getNodeById(String id) { if (id == null || StringUtil.noText(id)) { return null; } - - for (Node node : this.nodes) { - if (id.equals(node.getId())) { - return node; - } - } - - return null; + return graphIndex().nodeById.get(id); } @@ -138,49 +156,33 @@ public class ChainDefinition implements Serializable { this.edges = new ArrayList<>(); } this.edges.add(edge); - -// boolean findSource = false, findTarget = false; -// for (Node node : this.nodes) { -// if (node.getId().equals(edge.getSource())) { -// node.addOutwardEdge(edge); -// findSource = true; -// } else if (node.getId().equals(edge.getTarget())) { -// node.addInwardEdge(edge); -// findTarget = true; -// } -// if (findSource && findTarget) { -// break; -// } -// } + invalidateGraphIndex(); } + /** + * 按 ID 获取边。 + * + * @param edgeId 边 ID + * @return 对应边,不存在时返回 {@code null} + */ public Edge getEdgeById(String edgeId) { - for (Edge edge : this.edges) { - if (edgeId.equals(edge.getId())) { - return edge; - } + if (StringUtil.noText(edgeId)) { + return null; } - return null; + return graphIndex().edgeById.get(edgeId); } + /** + * 获取没有入边的开始节点。 + * + * @return 保持定义顺序的开始节点副本 + */ public List getStartNodes() { if (nodes == null || nodes.isEmpty()) { return null; } - - List result = new ArrayList<>(); - - for (Node node : nodes) { -// if (CollectionUtil.noItems(node.getInwardEdges())) { -// result.add(node); -// } - List inwardEdge = getInwardEdge(node.getId()); - if (inwardEdge == null || inwardEdge.isEmpty()) { - result.add(node); - } - } - return result; + return new ArrayList<>(graphIndex().startNodes); } @@ -198,6 +200,210 @@ public class ChainDefinition implements Serializable { return parameters; } + /** + * 使派生图索引失效。 + */ + private void invalidateGraphIndex() { + graphIndex = null; + } + + /** + * 获取当前节点和边对应的只读图索引。 + * + * @return 图索引 + */ + private GraphIndex graphIndex() { + GraphIndex current = graphIndex; + if (current != null) { + return current; + } + synchronized (this) { + current = graphIndex; + if (current == null) { + current = GraphIndex.build(nodes, edges); + graphIndex = current; + } + return current; + } + } + + /** + * 工作流定义的派生图索引。 + */ + private static final class GraphIndex { + private final Map nodeById; + private final Map edgeById; + private final Map> outwardEdgesByNode; + private final Map> inwardEdgesByNode; + private final Map> + loopChildrenByNode; + private final List startNodes; + + /** + * 创建不可变图索引。 + * + * @param nodeById 节点索引 + * @param edgeById 边索引 + * @param outwardEdgesByNode 出边索引 + * @param inwardEdgesByNode 入边索引 + * @param startNodes 开始节点 + */ + private GraphIndex(Map nodeById, + Map edgeById, + Map> outwardEdgesByNode, + Map> inwardEdgesByNode, + Map> + loopChildrenByNode, + List startNodes) { + this.nodeById = nodeById; + this.edgeById = edgeById; + this.outwardEdgesByNode = outwardEdgesByNode; + this.inwardEdgesByNode = inwardEdgesByNode; + this.loopChildrenByNode = loopChildrenByNode; + this.startNodes = startNodes; + } + + /** + * 根据节点和边构建索引。 + * + * @param nodes 节点列表 + * @param edges 边列表 + * @return 构建完成的图索引 + */ + private static GraphIndex build(List nodes, List edges) { + Map nodeById = new HashMap<>(); + Map edgeById = new HashMap<>(); + Map> outwardEdgesByNode = new HashMap<>(); + Map> inwardEdgesByNode = new HashMap<>(); + + if (nodes != null) { + for (Node node : nodes) { + if (node != null && StringUtil.hasText(node.getId())) { + // 保持旧实现遇到重复 ID 时返回第一个节点的行为。 + nodeById.putIfAbsent(node.getId(), node); + } + } + } + + if (edges != null) { + for (Edge edge : edges) { + if (edge == null) { + continue; + } + if (StringUtil.hasText(edge.getId())) { + edgeById.putIfAbsent(edge.getId(), edge); + } + if (StringUtil.hasText(edge.getSource())) { + outwardEdgesByNode + .computeIfAbsent(edge.getSource(), ignored -> new ArrayList<>()) + .add(edge); + } + if (StringUtil.hasText(edge.getTarget())) { + inwardEdgesByNode + .computeIfAbsent(edge.getTarget(), ignored -> new ArrayList<>()) + .add(edge); + } + } + } + + List startNodes = new ArrayList<>(); + if (nodes != null) { + for (Node node : nodes) { + if (node != null && !inwardEdgesByNode.containsKey(node.getId())) { + startNodes.add(node); + } + } + } + + freezeEdgeLists(outwardEdgesByNode); + freezeEdgeLists(inwardEdgesByNode); + Map> + loopChildrenByNode = new HashMap<>(); + for (Map.Entry> entry : + outwardEdgesByNode.entrySet()) { + List dispatches = new ArrayList<>(); + for (Edge edge : entry.getValue()) { + Node child = nodeById.get(edge.getTarget()); + if (child != null + && Objects.equals( + entry.getKey(), child.getParentId())) { + String branchId = edge.getId() == null + ? child.getId() + : edge.getId(); + dispatches.add(new LoopChildDispatch( + child, edge.getId(), branchId)); + } + } + if (!dispatches.isEmpty()) { + loopChildrenByNode.put( + entry.getKey(), + Collections.unmodifiableList(dispatches)); + } + } + return new GraphIndex( + Collections.unmodifiableMap(nodeById), + Collections.unmodifiableMap(edgeById), + Collections.unmodifiableMap(outwardEdgesByNode), + Collections.unmodifiableMap(inwardEdgesByNode), + Collections.unmodifiableMap(loopChildrenByNode), + Collections.unmodifiableList(startNodes)); + } + + /** + * 将邻接表中的边列表转换为只读列表。 + * + * @param edgesByNode 邻接表 + */ + private static void freezeEdgeLists(Map> edgesByNode) { + edgesByNode.replaceAll((ignored, value) -> Collections.unmodifiableList(value)); + } + } + + /** + * 循环直属分支的预编译调度描述。 + */ + public static final class LoopChildDispatch { + + private final Node node; + private final String edgeId; + private final String branchId; + + /** + * 创建调度描述。 + * + * @param node 目标节点 + * @param edgeId 边 ID + * @param branchId 稳定分支 ID + */ + private LoopChildDispatch( + Node node, String edgeId, String branchId) { + this.node = node; + this.edgeId = edgeId; + this.branchId = branchId; + } + + /** + * @return 目标节点 + */ + public Node getNode() { + return node; + } + + /** + * @return 边 ID + */ + public String getEdgeId() { + return edgeId; + } + + /** + * @return 稳定分支 ID + */ + public String getBranchId() { + return branchId; + } + } + @Override public String toString() { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainState.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainState.java index 21c0786..1370749 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainState.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ChainState.java @@ -37,8 +37,14 @@ import java.util.stream.Collectors; public class ChainState implements Serializable { + private static final long serialVersionUID = -7958235553581638052L; + private String instanceId; private String parentInstanceId; + /** + * 节点审计应归属的顶级执行实例 ID。 + */ + private String auditInstanceId; private String chainDefinitionId; private ConcurrentHashMap memory = new ConcurrentHashMap<>(); @@ -59,9 +65,18 @@ public class ChainState implements Serializable { private String message; private ExceptionSummary error; private long version; + /** + * 工作流实例首次启动时间,用于跨线程和跨进程执行时长保护。 + */ + private long startedAt; + /** + * 已进入业务执行的节点次数,用于全局执行预算。 + */ + private long childExecutionCount; public ChainState() { this.instanceId = UUID.randomUUID().toString(); + this.auditInstanceId = this.instanceId; this.status = ChainStatus.READY; this.computeCost = 0; } @@ -71,7 +86,15 @@ public class ChainState implements Serializable { } public void setInstanceId(String instanceId) { + String previousInstanceId = + this.instanceId; this.instanceId = instanceId; + if (auditInstanceId == null + || Objects.equals( + auditInstanceId, + previousInstanceId)) { + auditInstanceId = instanceId; + } } public String getParentInstanceId() { @@ -80,6 +103,30 @@ public class ChainState implements Serializable { public void setParentInstanceId(String parentInstanceId) { this.parentInstanceId = parentInstanceId; + if (StringUtil.hasText(parentInstanceId) + && Objects.equals( + auditInstanceId, instanceId)) { + auditInstanceId = null; + } + } + + /** + * 获取节点审计归属实例 ID。 + * + * @return 顶级审计实例 ID + */ + public String getAuditInstanceId() { + return auditInstanceId; + } + + /** + * 设置节点审计归属实例 ID。 + * + * @param auditInstanceId 顶级审计实例 ID + */ + public void setAuditInstanceId( + String auditInstanceId) { + this.auditInstanceId = auditInstanceId; } public String getChainDefinitionId() { @@ -127,7 +174,9 @@ public class ChainState implements Serializable { if (triggerEdgeIds == null) { triggerEdgeIds = new ArrayList<>(); } - triggerEdgeIds.add(edgeId); + if (!triggerEdgeIds.contains(edgeId)) { + triggerEdgeIds.add(edgeId); + } } public List getTriggerNodeIds() { @@ -142,7 +191,9 @@ public class ChainState implements Serializable { if (triggerNodeIds == null) { triggerNodeIds = new ArrayList<>(); } - triggerNodeIds.add(nodeId); + if (!triggerNodeIds.contains(nodeId)) { + triggerNodeIds.add(nodeId); + } } public List getUncheckedEdgeIds() { @@ -150,21 +201,31 @@ public class ChainState implements Serializable { } public void setUncheckedEdgeIds(List uncheckedEdgeIds) { - this.uncheckedEdgeIds = uncheckedEdgeIds; + this.uncheckedEdgeIds = uncheckedEdgeIds == null + ? new ArrayList<>() + : new ArrayList<>(new LinkedHashSet<>(uncheckedEdgeIds)); } - public void addUncheckedEdgeId(String edgeId) { + public boolean addUncheckedEdgeId(String edgeId) { if (uncheckedEdgeIds == null) { uncheckedEdgeIds = new ArrayList<>(); } + if (uncheckedEdgeIds.contains(edgeId)) { + return false; + } uncheckedEdgeIds.add(edgeId); + return true; } public boolean removeUncheckedEdgeId(String edgeId) { if (uncheckedEdgeIds == null) { return false; } - return uncheckedEdgeIds.remove(edgeId); + boolean removed = false; + while (uncheckedEdgeIds.remove(edgeId)) { + removed = true; + } + return removed; } public List getUncheckedNodeIds() { @@ -172,21 +233,31 @@ public class ChainState implements Serializable { } public void setUncheckedNodeIds(List uncheckedNodeIds) { - this.uncheckedNodeIds = uncheckedNodeIds; + this.uncheckedNodeIds = uncheckedNodeIds == null + ? new ArrayList<>() + : new ArrayList<>(new LinkedHashSet<>(uncheckedNodeIds)); } - public void addUncheckedNodeId(String nodeId) { + public boolean addUncheckedNodeId(String nodeId) { if (uncheckedNodeIds == null) { uncheckedNodeIds = new ArrayList<>(); } + if (uncheckedNodeIds.contains(nodeId)) { + return false; + } uncheckedNodeIds.add(nodeId); + return true; } public boolean removeUncheckedNodeId(String nodeId) { if (uncheckedNodeIds == null) { return false; } - return uncheckedNodeIds.remove(nodeId); + boolean removed = false; + while (uncheckedNodeIds.remove(nodeId)) { + removed = true; + } + return removed; } public Long getComputeCost() { @@ -281,6 +352,22 @@ public class ChainState implements Serializable { this.version = version; } + public long getStartedAt() { + return startedAt; + } + + public void setStartedAt(long startedAt) { + this.startedAt = startedAt; + } + + public long getChildExecutionCount() { + return childExecutionCount; + } + + public void setChildExecutionCount(long childExecutionCount) { + this.childExecutionCount = childExecutionCount; + } + public static ChainState fromJSON(String jsonString) { ParserConfig config = new ParserConfig(); config.putDeserializer(ChainState.class, new ChainDeserializer()); @@ -305,6 +392,8 @@ public class ChainState implements Serializable { this.status = ChainStatus.READY; this.message = null; this.error = null; + this.startedAt = 0L; + this.childExecutionCount = 0L; } @@ -340,10 +429,46 @@ public class ChainState implements Serializable { public Object resolveValue(String path) { + return resolveValue(path, false); + } + + /** + * 解析参数路径,并可在直接命中时保留大型结果引用。 + * + * @param path 参数路径 + * @param preserveDirectReference 是否保留直接命中的引用 + * @return 参数值 + */ + private Object resolveValue( + String path, + boolean preserveDirectReference) { Object result = MapUtil.getByPath(getMemory(), path); if (result == null) result = MapUtil.getByPath(getEnvironment(), path); // if (result == null) result = MapUtil.getByPath(getTriggerVariables(), path); - return result; + Chain chain = Chain.currentChain(); + if (result != null || chain == null || memory == null || path == null) { + return chain == null + || preserveDirectReference + ? result + : chain.resolveResultReferences(result); + } + + // MapUtil 无法直接穿透轻量引用,先解析最长命中的作用域值,再继续解析剩余路径。 + String[] parts = path.split("\\."); + for (int length = parts.length - 1; length > 0; length--) { + String prefix = String.join(".", Arrays.copyOf(parts, length)); + Object referenced = memory.get(prefix); + if (referenced == null) { + continue; + } + Object resolved = chain.resolveResultReferences(referenced); + String remaining = String.join( + ".", Arrays.copyOfRange(parts, length, parts.length)); + return MapUtil.getByPath( + Collections.singletonMap("value", resolved), + "value." + remaining); + } + return null; } public Map resolveParameters(Node node) { @@ -381,7 +506,28 @@ public class ChainState implements Serializable { * @return 模板渲染上下文列表 */ public List> buildTemplateRootMaps(Map formatArgs) { - return Arrays.asList(getMemory(), formatArgs, getEnvMap()); + Chain chain = Chain.currentChain(); + Map runtimeMemory = getMemory(); + if (chain != null && runtimeMemory != null && !runtimeMemory.isEmpty()) { + runtimeMemory = new LazyReferenceMap( + runtimeMemory, chain); + } + return Arrays.asList(runtimeMemory, formatArgs, getEnvMap()); + } + + /** + * 构建审计参数使用的惰性模板上下文。 + * + *

仅在模板实际读取某个 memory 顶级值时还原其中的轻量引用, + * 避免无关固定参数同步物化大型结果。

+ * + * @param formatArgs 当前节点参与模板渲染的参数 + * @return 惰性模板上下文列表 + */ + private List> + buildLazyTemplateRootMaps( + Map formatArgs) { + return buildTemplateRootMaps(formatArgs); } /** @@ -403,23 +549,76 @@ public class ChainState implements Serializable { } public Map resolveParameters(Node node, List parameters, Map formatArgs, boolean ignoreRequired) { + return resolveParameters( + node, + parameters, + formatArgs, + ignoreRequired, + false); + } + + /** + * 解析节点审计输入,直接引用保持轻量形式,由审计消费者异步还原。 + * + * @param node 当前节点 + * @return 兼容既有输入字段结构的参数快照 + */ + public Map resolveParametersPreservingReferences( + Node node) { + return resolveParameters( + node, + node.getParameters(), + null, + false, + true); + } + + /** + * 解析节点参数。 + * + * @param node 当前节点 + * @param parameters 参数定义 + * @param formatArgs 模板附加参数 + * @param ignoreRequired 是否忽略必填校验 + * @param preserveDirectReferences 是否保留直接结果引用 + * @return 已解析参数 + */ + private Map resolveParameters( + Node node, + List parameters, + Map formatArgs, + boolean ignoreRequired, + boolean preserveDirectReferences) { if (parameters == null || parameters.isEmpty()) { return Collections.emptyMap(); } Map variables = new LinkedHashMap<>(); List suspendParameters = null; + List> templateRootMaps = null; for (Parameter parameter : parameters) { RefType refType = parameter.getRefType(); Object value = null; if (refType == RefType.FIXED) { + if (templateRootMaps == null) { + templateRootMaps = + preserveDirectReferences + ? buildLazyTemplateRootMaps( + formatArgs) + : buildTemplateRootMaps( + formatArgs); + } value = TextTemplate.of(parameter.getValue()) - .formatToString(buildTemplateRootMaps(formatArgs)); + .formatToString(templateRootMaps); } else if (refType == RefType.REF) { - value = this.resolveValue(parameter.getRef()); + value = this.resolveValue( + parameter.getRef(), + preserveDirectReferences); } // 单节点执行时,参数只会传入 name 内容。 if (value == null) { - value = this.resolveValue(parameter.getName()); + value = this.resolveValue( + parameter.getName(), + preserveDirectReferences); } if (value == null && parameter.getDefaultValue() != null) { @@ -475,6 +674,166 @@ public class ChainState implements Serializable { return variables; } + /** + * 按实际访问惰性还原顶级 memory 值的只读映射。 + */ + private static final class LazyReferenceMap + extends AbstractMap { + + private final Map delegate; + private final Chain chain; + /** + * 同一模板渲染内已经还原的顶级值,避免重复引用触发重复分块读取。 + */ + private final Map resolvedValues = + new HashMap<>(); + + /** + * 创建惰性引用映射。 + * + * @param delegate 原始运行时 memory + * @param chain 当前工作流链路 + */ + private LazyReferenceMap( + Map delegate, + Chain chain) { + this.delegate = delegate; + this.chain = chain; + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized Object get(Object key) { + if (!delegate.containsKey(key)) { + return null; + } + if (resolvedValues.containsKey(key)) { + return resolvedValues.get(key); + } + Object resolved = + chain.resolveResultReferences( + delegate.get(key)); + resolvedValues.put(key, resolved); + return resolved; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean containsKey(Object key) { + return delegate.containsKey(key); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isEmpty() { + return delegate.isEmpty(); + } + + /** + * {@inheritDoc} + */ + @Override + public int size() { + return delegate.size(); + } + + /** + * {@inheritDoc} + */ + @Override + public Set keySet() { + return Collections.unmodifiableSet( + delegate.keySet()); + } + + /** + * {@inheritDoc} + */ + @Override + public Set> entrySet() { + Set keys = keySet(); + return new AbstractSet<>() { + @Override + public Iterator> + iterator() { + Iterator iterator = + keys.iterator(); + return new Iterator<>() { + @Override + public boolean hasNext() { + return iterator + .hasNext(); + } + + @Override + public Entry + next() { + String key = + iterator.next(); + return lazyEntry(key); + } + }; + } + + @Override + public int size() { + return keys.size(); + } + }; + } + + /** + * 创建仅在读取值时还原引用的不可变条目。 + * + * @param key memory 键 + * @return 惰性条目 + */ + private Entry lazyEntry( + String key) { + return new Entry<>() { + @Override + public String getKey() { + return key; + } + + @Override + public Object getValue() { + return LazyReferenceMap.this + .get(key); + } + + @Override + public Object setValue(Object value) { + throw new UnsupportedOperationException( + "read-only runtime memory"); + } + + @Override + public boolean equals(Object value) { + return value instanceof Entry entry + && Objects.equals( + key, entry.getKey()) + && Objects.equals( + getValue(), + entry.getValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(key) + ^ Objects.hashCode( + getValue()); + } + }; + } + } + public static class ChainSerializer implements ObjectSerializer { @Override @@ -513,6 +872,8 @@ public class ChainState implements Serializable { ", message='" + message + '\'' + ", error=" + error + ", version=" + version + + ", startedAt=" + startedAt + + ", childExecutionCount=" + childExecutionCount + '}'; } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Edge.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Edge.java index b559e63..8ea7b31 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Edge.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Edge.java @@ -16,7 +16,14 @@ package com.easyagents.flow.core.chain; -public class Edge { +import java.io.Serializable; + +/** + * 工作流节点之间的有向边定义。 + */ +public class Edge implements Serializable { + private static final long serialVersionUID = 1L; + private String id; private String source; private String target; diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EdgeCondition.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EdgeCondition.java index 3629807..7a0c621 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EdgeCondition.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/EdgeCondition.java @@ -16,10 +16,22 @@ package com.easyagents.flow.core.chain; +import java.io.Serializable; import java.util.Map; -public interface EdgeCondition { +/** + * 工作流边的执行条件。 + */ +public interface EdgeCondition extends Serializable { + /** + * 判断边是否允许继续执行。 + * + * @param chain 当前工作流实例 + * @param edge 待检查的边 + * @param executeResult 上游节点执行结果 + * @return 允许执行时返回 {@code true} + */ boolean check(Chain chain, Edge edge, Map executeResult); } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ExceptionSummary.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ExceptionSummary.java index 60eb033..8438cd5 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ExceptionSummary.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/ExceptionSummary.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.io.StringWriter; public class ExceptionSummary implements Serializable { + private static final long serialVersionUID = 1L; private String exceptionClass; private String message; @@ -134,4 +135,3 @@ public class ExceptionSummary implements Serializable { this.timestamp = timestamp; } } - diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Node.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Node.java index 62fcc0b..643a1da 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Node.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Node.java @@ -25,7 +25,12 @@ import java.util.List; import java.util.Map; public abstract class Node implements Serializable { + private static final long serialVersionUID = 1L; private static final Logger log = org.slf4j.LoggerFactory.getLogger(Node.class); + /** 可配置的最小循环次数。 */ + public static final int MIN_LOOP_COUNT = 1; + /** 单个节点允许的最大循环次数。 */ + public static final int MAX_LOOP_COUNT = 300; protected String id; protected String parentId; @@ -42,7 +47,7 @@ public abstract class Node implements Serializable { protected boolean loopEnable = false; // 是否启用循环执行 protected long loopIntervalMs = 3000; // 循环间隔时间(毫秒) protected NodeCondition loopBreakCondition; // 跳出循环的条件 - protected int maxLoopCount = 0; // 0 表示不限制循环次数 + protected int maxLoopCount = MIN_LOOP_COUNT; // 循环总执行次数,取值范围 1~300 protected boolean retryEnable = false; protected boolean resetRetryCountAfterNormal = false; @@ -158,7 +163,22 @@ public abstract class Node implements Serializable { return maxLoopCount; } + /** + * 设置节点循环的总执行次数。 + * + * @param maxLoopCount 总执行次数,范围为 1~300 + * @throws IllegalArgumentException 循环次数超出允许范围 + */ public void setMaxLoopCount(int maxLoopCount) { + if (maxLoopCount < MIN_LOOP_COUNT || maxLoopCount > MAX_LOOP_COUNT) { + throw new IllegalArgumentException( + "maxLoopCount must be between " + + MIN_LOOP_COUNT + + " and " + + MAX_LOOP_COUNT + + ", but was " + + maxLoopCount); + } this.maxLoopCount = maxLoopCount; } @@ -237,7 +257,12 @@ public abstract class Node implements Serializable { protected long doCalculateComputeCost(String expr, Chain chain, Map result) { // Map parameterValues = chain.getState().getParameterValuesOnly(this, this.getParameters(), null); - Map parameterValues = chain.getState().resolveParameters(this, this.getParameters(), null,true); + Map parameterValues = + chain.getExecutionState().resolveParameters( + this, + this.getParameters(), + null, + true); Map newMap = new HashMap<>(result); newMap.putAll(parameterValues); return JsConditionUtil.evalLong(expr, chain, newMap); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeCondition.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeCondition.java index f3adc40..e1e463a 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeCondition.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeCondition.java @@ -16,10 +16,22 @@ package com.easyagents.flow.core.chain; +import java.io.Serializable; import java.util.Map; -public interface NodeCondition { +/** + * 工作流节点的执行条件。 + */ +public interface NodeCondition extends Serializable { + /** + * 判断节点是否允许继续执行。 + * + * @param chain 当前工作流实例 + * @param context 当前节点状态 + * @param executeResult 上一次执行结果 + * @return 允许执行时返回 {@code true} + */ boolean check(Chain chain, NodeState context, Map executeResult); } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeState.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeState.java index ab1a7de..3965b29 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeState.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeState.java @@ -20,10 +20,11 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; public class NodeState implements Serializable { + private static final long serialVersionUID = -6727481826462129573L; + private String nodeId; private String chainInstanceId; @@ -39,6 +40,11 @@ public class NodeState implements Serializable { private AtomicInteger executeCount = new AtomicInteger(0); private List executeEdgeIds = new ArrayList<>(); + /** + * 当前节点生命周期对应的稳定业务尝试键。 + */ + private String executionAttemptKey; + ExceptionSummary error; private long version; @@ -135,6 +141,26 @@ public class NodeState implements Serializable { this.executeEdgeIds = executeEdgeIds; } + /** + * 获取当前节点生命周期的稳定业务尝试键。 + * + * @return 稳定业务尝试键 + */ + public String getExecutionAttemptKey() { + return executionAttemptKey; + } + + /** + * 设置当前节点生命周期的稳定业务尝试键。 + * + * @param executionAttemptKey 稳定业务尝试键 + */ + public void setExecutionAttemptKey( + String executionAttemptKey) { + this.executionAttemptKey = + executionAttemptKey; + } + public ExceptionSummary getError() { return error; } @@ -158,10 +184,16 @@ public class NodeState implements Serializable { return true; } - List shouldBeTriggerIds = inwardEdges.stream().map(Edge::getId).collect(Collectors.toList()); - List triggerEdgeIds = this.triggerEdgeIds; - return triggerEdgeIds.size() >= shouldBeTriggerIds.size() - && shouldBeTriggerIds.parallelStream().allMatch(triggerEdgeIds::contains); + if (triggerEdgeIds.size() < inwardEdges.size()) { + return false; + } + java.util.Set triggeredEdges = new java.util.HashSet<>(triggerEdgeIds); + for (Edge inwardEdge : inwardEdges) { + if (!triggeredEdges.contains(inwardEdge.getId())) { + return false; + } + } + return true; } public void recordTrigger(String fromEdgeId) { @@ -169,7 +201,9 @@ public class NodeState implements Serializable { if (fromEdgeId == null) { fromEdgeId = "none"; } - triggerEdgeIds.add(fromEdgeId); + if (!triggerEdgeIds.contains(fromEdgeId)) { + triggerEdgeIds.add(fromEdgeId); + } } public void recordExecute(String fromEdgeId) { @@ -177,6 +211,7 @@ public class NodeState implements Serializable { if (fromEdgeId == null) { fromEdgeId = "none"; } + executeEdgeIds.clear(); executeEdgeIds.add(fromEdgeId); } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeValidator.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeValidator.java index 0cb2a51..ac2e4d0 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeValidator.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/NodeValidator.java @@ -16,6 +16,18 @@ package com.easyagents.flow.core.chain; -public interface NodeValidator { +import java.io.Serializable; + +/** + * 工作流节点定义校验器。 + */ +public interface NodeValidator extends Serializable { + + /** + * 校验节点定义。 + * + * @param node 待校验节点 + * @return 校验结果 + */ NodeValidResult validate(Node node); } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java index d3eedf2..97f4bc9 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; public class Parameter implements Serializable, Cloneable { + private static final long serialVersionUID = 1L; protected String id; protected String name; protected String description; diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeEndEvent.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeEndEvent.java index 97a1c30..cfecc8e 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeEndEvent.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeEndEvent.java @@ -18,40 +18,133 @@ package com.easyagents.flow.core.chain.event; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.Node; +import com.easyagents.flow.core.chain.NodeStatus; import java.util.Map; +/** + * 节点结束执行事件。 + */ public class NodeEndEvent extends BaseEvent { private final Node node; private final Map result; private final Throwable error; + private final NodeStatus status; + private final String executionAttemptKey; + /** + * 创建节点结束事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + * @param result 节点输出 + * @param error 节点异常 + */ public NodeEndEvent(Chain chain, Node node, Map result, Throwable error) { + this(chain, node, result, error, null, null); + } + + /** + * 创建携带不可变业务尝试键的节点结束事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + * @param result 节点输出 + * @param error 节点异常 + * @param executionAttemptKey 节点本次业务尝试键 + */ + public NodeEndEvent(Chain chain, + Node node, + Map result, + Throwable error, + String executionAttemptKey) { + this( + chain, + node, + result, + error, + null, + executionAttemptKey); + } + + /** + * 创建携带不可变节点终态和业务尝试键的节点结束事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + * @param result 节点输出 + * @param error 节点异常 + * @param status 节点本次业务尝试终态 + * @param executionAttemptKey 节点本次业务尝试键 + */ + public NodeEndEvent(Chain chain, + Node node, + Map result, + Throwable error, + NodeStatus status, + String executionAttemptKey) { super(chain); this.node = node; this.result = result; this.error = error; + this.status = status; + this.executionAttemptKey = executionAttemptKey; } + /** + * 获取当前节点。 + * + * @return 当前节点 + */ public Node getNode() { return node; } + /** + * 获取节点输出。 + * + * @return 节点输出 + */ public Map getResult() { return result; } + /** + * 获取节点异常。 + * + * @return 节点异常;成功时为 {@code null} + */ public Throwable getError() { return error; } + /** + * 获取事件创建时捕获的节点终态。 + * + * @return 节点终态;旧调用方未提供时为 {@code null} + */ + public NodeStatus getStatus() { + return status; + } + + /** + * 获取事件创建时捕获的业务尝试键。 + * + * @return 业务尝试键;旧调用方未提供时为 {@code null} + */ + public String getExecutionAttemptKey() { + return executionAttemptKey; + } + @Override public String toString() { return "NodeEndEvent{" + "node=" + node + ", result=" + result + ", error=" + error + + ", status=" + status + + ", executionAttemptKey='" + executionAttemptKey + '\'' + ", chain=" + chain + '}'; } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeStartEvent.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeStartEvent.java index 9e4358b..e54b4f6 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeStartEvent.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/NodeStartEvent.java @@ -18,25 +18,123 @@ package com.easyagents.flow.core.chain.event; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.Node; +import com.easyagents.flow.core.chain.NodeStatus; +/** + * 节点开始执行事件。 + */ public class NodeStartEvent extends BaseEvent { private final Node node; + private final String executionAttemptKey; + private final NodeStatus status; + private final String auditInstanceId; + /** + * 创建节点开始事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + */ public NodeStartEvent(Chain chain, Node node) { - super(chain); - this.node = node; + this(chain, node, null, null, + chain.getStateInstanceId()); } + /** + * 创建携带不可变业务尝试键的节点开始事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + * @param executionAttemptKey 节点本次业务尝试键 + */ + public NodeStartEvent(Chain chain, + Node node, + String executionAttemptKey) { + this(chain, node, executionAttemptKey, null, + chain.getStateInstanceId()); + } + + /** + * 创建携带不可变业务尝试键和节点状态的开始事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + * @param executionAttemptKey 节点本次业务尝试键 + * @param status 事件创建时的节点状态 + */ + public NodeStartEvent(Chain chain, + Node node, + String executionAttemptKey, + NodeStatus status) { + this(chain, node, executionAttemptKey, status, + chain.getStateInstanceId()); + } + + /** + * 创建携带完整不可变审计上下文的节点开始事件。 + * + * @param chain 当前工作流 + * @param node 当前节点 + * @param executionAttemptKey 节点本次业务尝试键 + * @param status 事件创建时的节点状态 + * @param auditInstanceId 节点审计应关联的顶级执行实例 ID + */ + public NodeStartEvent(Chain chain, + Node node, + String executionAttemptKey, + NodeStatus status, + String auditInstanceId) { + super(chain); + this.node = node; + this.executionAttemptKey = executionAttemptKey; + this.status = status; + this.auditInstanceId = auditInstanceId; + } + + /** + * 获取当前节点。 + * + * @return 当前节点 + */ public Node getNode() { return node; } + /** + * 获取事件创建时捕获的业务尝试键。 + * + * @return 业务尝试键;旧调用方未提供时为 {@code null} + */ + public String getExecutionAttemptKey() { + return executionAttemptKey; + } + + /** + * 获取事件创建时捕获的节点状态。 + * + * @return 节点状态;旧调用方未提供时为 {@code null} + */ + public NodeStatus getStatus() { + return status; + } + + /** + * 获取节点审计关联的顶级执行实例 ID。 + * + * @return 顶级执行实例 ID + */ + public String getAuditInstanceId() { + return auditInstanceId; + } @Override public String toString() { return "NodeStartEvent{" + "node=" + node + + ", executionAttemptKey='" + executionAttemptKey + '\'' + + ", status=" + status + + ", auditInstanceId='" + auditInstanceId + '\'' + ", chain=" + chain + '}'; } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainDefinitionSnapshotRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainDefinitionSnapshotRepository.java new file mode 100644 index 0000000..5143e28 --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainDefinitionSnapshotRepository.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + * Licensed under the GNU Lesser General Public License (LGPL), Version 3.0. + */ +package com.easyagents.flow.core.chain.repository; + +import com.easyagents.flow.core.chain.ChainDefinition; + +/** + * 工作流实例级定义快照仓储。 + */ +public interface ChainDefinitionSnapshotRepository { + + /** + * 保存实例启动时的定义快照。 + * + * @param instanceId 工作流实例 ID + * @param definition 定义快照 + */ + void save(String instanceId, ChainDefinition definition); + + /** + * 加载实例启动时的定义快照。 + * + * @param instanceId 工作流实例 ID + * @return 定义快照;不存在时返回 null + */ + ChainDefinition load(String instanceId); + + /** + * 删除定义快照。 + * + * @param instanceId 工作流实例 ID + */ + void remove(String instanceId); +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainLock.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainLock.java index 1bbcf17..95bf703 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainLock.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainLock.java @@ -25,9 +25,27 @@ public interface ChainLock extends AutoCloseable { */ boolean isAcquired(); + /** + * 锁是否仍由当前 owner 持有。 + * + * @return 锁仍有效时为 true + */ + default boolean isValid() { + return isAcquired(); + } + + /** + * 获取本次锁持有期对应的 fencing token。 + * + * @return 分布式仓储生成的单实例单调递增 token;本地锁返回 {@code 0} + */ + default long getFencingToken() { + return 0L; + } + /** * 释放锁(幂等) */ @Override void close(); -} \ No newline at end of file +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateField.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateField.java index 1509d23..b06cb9c 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateField.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateField.java @@ -31,8 +31,12 @@ public enum ChainStateField { ENVIRONMENT, CHILD_STATE_IDS, PARENT_INSTANCE_ID, + AUDIT_INSTANCE_ID, TRIGGER_NODE_IDS, TRIGGER_EDGE_IDS, UNCHECKED_EDGE_IDS, - UNCHECKED_NODE_IDS; + UNCHECKED_NODE_IDS, + STARTED_AT, + CHILD_EXECUTION_COUNT, + VERSION; } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateRepository.java index 45df7c7..5d2a543 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateRepository.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/ChainStateRepository.java @@ -24,8 +24,70 @@ public interface ChainStateRepository { ChainState load(String instanceId); + /** + * 轻量读取工作流状态版本。 + * + *

分布式仓储应覆盖本方法并只读取版本字段,避免节点状态提交前反序列化完整 + * 工作流热状态。

+ * + * @param instanceId 工作流实例 ID + * @return 当前版本;状态不存在时返回 {@code null} + */ + default Long loadVersion(String instanceId) { + ChainState state = load(instanceId); + return state == null ? null : state.getVersion(); + } + + /** + * 创建工作流实例状态。 + * + * @param instanceId 工作流实例 ID + * @return 已存在或新创建的状态 + */ + default ChainState create(String instanceId) { + return load(instanceId); + } + boolean tryUpdate(ChainState newState, EnumSet fields); + /** + * 在当前实例锁 fencing token 仍有效时提交状态。 + * + *

单进程仓储可沿用普通乐观锁;分布式仓储应覆盖本方法并在同一原子操作中校验 + * token。

+ * + * @param newState 待提交状态 + * @param fields 变化字段 + * @param fencingToken 当前实例锁 token;本地仓储调用为 {@code 0} + * @return 提交成功时为 {@code true} + */ + default boolean tryUpdate( + ChainState newState, EnumSet fields, long fencingToken) { + return tryUpdate(newState, fields); + } + + /** + * 在实例锁和当前触发器认领租约均有效时提交状态。 + * + *

分布式仓储应在同一原子操作中校验实例锁 fencing token 与 claim generation, + * 同时拒绝锁过期后的旧执行者和租约过期后的旧 owner。

+ * + * @param newState 待提交状态 + * @param fields 变化字段 + * @param lockFencingToken 当前实例锁 token;本地仓储调用为 {@code 0} + * @param claimId 当前触发器 ID;非触发器调用为 {@code null} + * @param claimGeneration 当前认领代际;非触发器调用为 {@code 0} + * @return 提交成功时为 {@code true} + */ + default boolean tryUpdate( + ChainState newState, + EnumSet fields, + long lockFencingToken, + String claimId, + long claimGeneration) { + return tryUpdate(newState, fields, lockFencingToken); + } + /** * 获取指定 instanceId 的分布式锁 * diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainDefinitionSnapshotRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainDefinitionSnapshotRepository.java new file mode 100644 index 0000000..ea9cecc --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainDefinitionSnapshotRepository.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + * Licensed under the GNU Lesser General Public License (LGPL), Version 3.0. + */ +package com.easyagents.flow.core.chain.repository; + +import com.easyagents.flow.core.chain.ChainDefinition; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 进程内工作流定义快照仓储。 + */ +public class InMemoryChainDefinitionSnapshotRepository + implements ChainDefinitionSnapshotRepository { + + private final Map snapshots = new ConcurrentHashMap<>(); + + /** + * {@inheritDoc} + */ + @Override + public void save(String instanceId, ChainDefinition definition) { + snapshots.put(instanceId, definition); + } + + /** + * {@inheritDoc} + */ + @Override + public ChainDefinition load(String instanceId) { + return snapshots.get(instanceId); + } + + /** + * {@inheritDoc} + */ + @Override + public void remove(String instanceId) { + snapshots.remove(instanceId); + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainStateRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainStateRepository.java index 65f1d08..5fb9755 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainStateRepository.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryChainStateRepository.java @@ -16,24 +16,41 @@ package com.easyagents.flow.core.chain.repository; import com.easyagents.flow.core.chain.ChainState; -import com.easyagents.flow.core.util.MapUtil; import java.util.EnumSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * 进程内工作流状态仓储。 + */ public class InMemoryChainStateRepository implements ChainStateRepository { private static final Map chainStateMap = new ConcurrentHashMap<>(); + /** + * {@inheritDoc} + */ @Override public ChainState load(String instanceId) { - return MapUtil.computeIfAbsent(chainStateMap, instanceId, k -> { + // 保留进程内仓储原有的惰性初始化语义,兼容直接构造 Chain 的调用方式。 + return create(instanceId); + } + + /** + * {@inheritDoc} + */ + @Override + public ChainState create(String instanceId) { + return chainStateMap.computeIfAbsent(instanceId, ignored -> { ChainState state = new ChainState(); state.setInstanceId(instanceId); return state; }); } + /** + * {@inheritDoc} + */ @Override public boolean tryUpdate(ChainState chainState, EnumSet fields) { chainStateMap.put(chainState.getInstanceId(), chainState); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryLoopResultRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryLoopResultRepository.java new file mode 100644 index 0000000..55b0644 --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryLoopResultRepository.java @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + *

+ * Licensed under the GNU Lesser General Public License (LGPL) ,Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl-3.0.txt + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.flow.core.chain.repository; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 进程内循环累计结果仓储,适用于单机运行和测试。 + */ +public class InMemoryLoopResultRepository implements LoopResultRepository { + + private final Map>> results = new ConcurrentHashMap<>(); + private final Map> inputs = new ConcurrentHashMap<>(); + + /** + * {@inheritDoc} + */ + @Override + public int storeInput(String resultId, Iterable items) { + List stored = new ArrayList<>(); + for (Object item : items) { + stored.add(item); + } + List existing = inputs.putIfAbsent(resultId, stored); + return existing == null ? stored.size() : existing.size(); + } + + /** + * {@inheritDoc} + */ + @Override + public Object loadInputItem(String resultId, int index) { + List stored = inputs.get(resultId); + if (stored == null) { + throw new IllegalStateException("Loop input not found: " + resultId); + } + return stored.get(index); + } + + /** + * {@inheritDoc} + */ + @Override + public void removeInput(String resultId) { + inputs.remove(resultId); + } + + /** + * {@inheritDoc} + */ + @Override + public void append(String resultId, int iterationIndex, Map outputValues) { + if (outputValues == null || outputValues.isEmpty()) { + return; + } + Map> result = results.computeIfAbsent( + resultId, ignored -> Collections.synchronizedMap(new LinkedHashMap<>())); + synchronized (result) { + for (Map.Entry entry : outputValues.entrySet()) { + List values = result.computeIfAbsent(entry.getKey(), ignored -> new ArrayList<>()); + if (values.size() != iterationIndex) { + throw new IllegalStateException("Unexpected loop result index: " + iterationIndex); + } + values.add(entry.getValue()); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public Map load(String resultId, int iterationCount, List outputNames) { + Map> result = results.get(resultId); + Map snapshot = new LinkedHashMap<>(); + if (iterationCount == 0 || outputNames == null || outputNames.isEmpty()) { + return snapshot; + } + if (result == null) { + throw new IllegalStateException("Loop result not found: " + resultId); + } + synchronized (result) { + for (String outputName : outputNames) { + List values = result.get(outputName); + if (values == null || values.size() != iterationCount) { + throw new IllegalStateException("Incomplete loop result: " + outputName); + } + snapshot.put(outputName, new ArrayList<>(values)); + } + } + return snapshot; + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryNodeStateRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryNodeStateRepository.java index 3bbfd38..231991b 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryNodeStateRepository.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/InMemoryNodeStateRepository.java @@ -16,20 +16,33 @@ package com.easyagents.flow.core.chain.repository; import com.easyagents.flow.core.chain.NodeState; -import com.easyagents.flow.core.util.MapUtil; import java.util.EnumSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * 进程内节点状态仓储。 + */ public class InMemoryNodeStateRepository implements NodeStateRepository { private static final Map chainStateMap = new ConcurrentHashMap<>(); + /** + * {@inheritDoc} + */ @Override public NodeState load(String instanceId, String nodeId) { - String key = instanceId + "." + nodeId; - return MapUtil.computeIfAbsent(chainStateMap, key, k -> { + // 保留进程内仓储原有的惰性初始化语义,避免改变既有直接读取行为。 + return create(instanceId, nodeId, 0L); + } + + /** + * {@inheritDoc} + */ + @Override + public NodeState create(String instanceId, String nodeId, long chainStateVersion) { + return chainStateMap.computeIfAbsent(key(instanceId, nodeId), ignored -> { NodeState nodeState = new NodeState(); nodeState.setChainInstanceId(instanceId); nodeState.setNodeId(nodeId); @@ -37,9 +50,23 @@ public class InMemoryNodeStateRepository implements NodeStateRepository { }); } + /** + * {@inheritDoc} + */ @Override public boolean tryUpdate(NodeState newState, EnumSet fields, long version) { - chainStateMap.put(newState.getChainInstanceId() + "." + newState.getNodeId(), newState); + chainStateMap.put(key(newState.getChainInstanceId(), newState.getNodeId()), newState); return true; } + + /** + * 构建进程内节点状态键。 + * + * @param instanceId 工作流实例 ID + * @param nodeId 节点 ID + * @return 节点状态键 + */ + private String key(String instanceId, String nodeId) { + return instanceId + "." + nodeId; + } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopInputReference.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopInputReference.java new file mode 100644 index 0000000..73b921b --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopInputReference.java @@ -0,0 +1,62 @@ +package com.easyagents.flow.core.chain.repository; + +import java.io.Serializable; +import java.util.Objects; + +/** + * 已分块保存的循环输入轻量引用。 + * + *

循环节点按序读取分块;其他业务节点在参数读取边界会透明还原为与原输入等价的列表。

+ */ +public final class LoopInputReference implements Serializable { + + private static final long serialVersionUID = 1L; + private static final String REFERENCE_TYPE = + "easyflow.loop-input.v1"; + + private final String resultId; + private final int itemCount; + + /** + * 创建循环输入引用。 + * + * @param resultId 循环输入结果 ID + * @param itemCount 输入元素数量 + */ + public LoopInputReference(String resultId, int itemCount) { + this.resultId = Objects.requireNonNull( + resultId, "resultId must not be null"); + if (itemCount < 0) { + throw new IllegalArgumentException( + "itemCount must not be negative"); + } + this.itemCount = itemCount; + } + + /** + * 获取循环输入结果 ID。 + * + * @return 结果 ID + */ + public String getResultId() { + return resultId; + } + + /** + * 获取输入元素数量。 + * + * @return 元素数量 + */ + public int getItemCount() { + return itemCount; + } + + /** + * 获取跨异步审计边界使用的稳定引用类型。 + * + * @return 引用类型 + */ + public String getReferenceType() { + return REFERENCE_TYPE; + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java new file mode 100644 index 0000000..cd1a740 --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java @@ -0,0 +1,52 @@ +package com.easyagents.flow.core.chain.repository; + +import java.io.Serializable; +import java.util.Objects; + +/** + * 循环累计输出的轻量引用,避免完整列表回写到高频热状态。 + */ +public final class LoopResultReference implements Serializable { + + private static final long serialVersionUID = 1L; + private static final String REFERENCE_TYPE = + "easyflow.loop-result.v1"; + + private final String resultId; + private final int iterationCount; + private final String outputName; + + /** + * 创建循环结果引用。 + * + * @param resultId 循环结果 ID + * @param iterationCount 迭代次数 + * @param outputName 输出名称 + */ + public LoopResultReference(String resultId, int iterationCount, String outputName) { + this.resultId = Objects.requireNonNull(resultId, "resultId must not be null"); + this.iterationCount = iterationCount; + this.outputName = Objects.requireNonNull(outputName, "outputName must not be null"); + } + + public String getResultId() { + return resultId; + } + + public int getIterationCount() { + return iterationCount; + } + + public String getOutputName() { + return outputName; + } + + /** + * 获取跨异步审计边界使用的稳定引用类型。 + * + * @return 引用类型 + */ + public String getReferenceType() { + return REFERENCE_TYPE; + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java new file mode 100644 index 0000000..55cd40c --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java @@ -0,0 +1,486 @@ +/** + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + *

+ * Licensed under the GNU Lesser General Public License (LGPL) ,Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl-3.0.txt + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.flow.core.chain.repository; + +import com.easyagents.flow.core.chain.runtime.ExecutionBudgetExceededException; + +import java.util.Iterator; +import java.util.List; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** + * 循环节点累计结果仓储。 + *

+ * 累计结果独立于高频更新的节点状态保存,避免每轮迭代重复序列化全部历史结果。 + */ +public interface LoopResultRepository { + + /** + * 流式保存不可随机访问的循环输入。 + * + * @param resultId 循环结果 ID + * @param items 原始输入 + * @return 输入元素数量 + */ + int storeInput(String resultId, Iterable items); + + /** + * 在已启用的迭代预算内流式保存循环输入。 + * + *

实现会在读取第 {@code maxItems + 1} 个元素前终止,避免超大或无限 Iterable + * 先产生无界 I/O。具体仓储应在下游写入异常时清理已落盘的部分分块。

+ * + * @param resultId 循环结果 ID + * @param items 原始输入 + * @param maxItems 最大元素数;小于等于零表示不限制 + * @return 输入元素数量 + */ + default int storeInput(String resultId, Iterable items, long maxItems) { + if (maxItems <= 0L) { + return storeInput(resultId, items); + } + Iterable bounded = () -> new Iterator() { + private final Iterator delegate = items.iterator(); + private long count; + + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + + @Override + public Object next() { + if (count >= maxItems) { + throw new ExecutionBudgetExceededException( + "Loop iteration budget exceeded while storing input " + + resultId + + ": more than " + + maxItems); + } + count++; + return delegate.next(); + } + }; + return storeInput(resultId, bounded); + } + + /** + * 在实例锁和触发器认领均有效时流式保存循环输入。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @param resultId 循环结果 ID + * @param items 原始输入 + * @param maxItems 最大元素数;小于等于零表示不限制 + * @return 输入元素数量 + */ + default int storeInput( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + Iterable items, + long maxItems) { + return storeInput(resultId, items, maxItems); + } + + /** + * 在生产者主动推送数据时流式保存循环输入。 + * + *

缺省实现用于本地兼容仓储;分布式仓储应覆盖此方法并边接收边分块写入, + * 避免先构造完整列表。

+ * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @param resultId 循环结果 ID + * @param producer 输入生产者 + * @param maxItems 最大元素数;小于等于零表示不限制 + * @return 输入元素数量 + */ + default int storeProducedInput( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + InputProducer producer, + long maxItems) { + List items = new java.util.ArrayList<>(); + producer.produce(item -> { + if (maxItems > 0L + && items.size() >= maxItems) { + throw new ExecutionBudgetExceededException( + "Loop iteration budget exceeded while storing input " + + resultId + + ": more than " + + maxItems); + } + items.add(item); + }); + return storeInput( + instanceId, + lockFencingToken, + claimId, + claimGeneration, + resultId, + items, + 0L); + } + + /** + * 主动向循环输入仓储推送元素的生产者。 + */ + @FunctionalInterface + interface InputProducer { + + /** + * 生产并按原顺序推送输入元素。 + * + * @param sink 单元素接收器 + */ + void produce(Consumer sink); + } + + /** + * 按序号读取已保存的循环输入。 + * + * @param resultId 循环结果 ID + * @param index 从零开始的序号 + * @return 输入元素 + */ + Object loadInputItem(String resultId, int index); + + /** + * 在业务参数读取边界透明还原完整循环输入。 + * + * @param reference 循环输入引用 + * @return 与原输入顺序一致的列表 + */ + default List loadInput(LoopInputReference reference) { + List items = + new java.util.ArrayList<>( + reference.getItemCount()); + for (int index = 0; + index < reference.getItemCount(); + index++) { + items.add(loadInputItem( + reference.getResultId(), index)); + } + return items; + } + + /** + * 清理循环输入。 + * + * @param resultId 循环结果 ID + */ + default void removeInput(String resultId) { + } + + /** + * 释放指定循环结果的进程内活跃缓存。 + * + *

该操作不得删除已经持久化的输入、输出分块或改变结果引用语义,仅用于在 + * 循环完成后及时归还本机缓存空间。

+ * + * @param resultId 循环结果 ID + */ + default void releaseActiveCache(String resultId) { + } + + /** + * 在实例锁和触发器认领均有效时清理循环输入。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token + * @param claimId 当前触发器 ID + * @param claimGeneration 当前触发器认领代际 + * @param resultId 循环结果 ID + */ + default void removeInput( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId) { + removeInput(resultId); + } + + /** + * 追加一轮循环输出。 + * + * @param resultId 循环结果 ID + * @param iterationIndex 从零开始的迭代序号 + * @param outputValues 本轮输出 + */ + void append(String resultId, int iterationIndex, Map outputValues); + + /** + * 在当前触发器 fencing token 仍有效时追加循环输出。 + * + * @param instanceId 工作流实例 ID + * @param fencingToken 当前触发器 token;非持久化执行为 {@code 0} + * @param resultId 循环结果 ID + * @param iterationIndex 迭代序号 + * @param outputValues 本轮输出 + */ + default void append( + String instanceId, + long fencingToken, + String resultId, + int iterationIndex, + Map outputValues) { + append(resultId, iterationIndex, outputValues); + } + + /** + * 在实例锁和当前触发器认领租约均有效时追加循环输出。 + * + * @param instanceId 工作流实例 ID + * @param lockFencingToken 当前实例锁 token;本地仓储调用为 {@code 0} + * @param claimId 当前触发器 ID;非持久化执行为 {@code null} + * @param claimGeneration 当前认领代际;非持久化执行为 {@code 0} + * @param resultId 循环结果 ID + * @param iterationIndex 迭代序号 + * @param outputValues 本轮输出 + */ + default void append( + String instanceId, + long lockFencingToken, + String claimId, + long claimGeneration, + String resultId, + int iterationIndex, + Map outputValues) { + append(instanceId, lockFencingToken, resultId, iterationIndex, outputValues); + } + + /** + * 加载完整循环累计结果。 + * + * @param resultId 循环结果 ID + * @param iterationCount 已累计的迭代数 + * @param outputNames 输出名称,顺序与工作流定义一致 + * @return 按输出名称聚合的结果列表 + */ + Map load(String resultId, int iterationCount, List outputNames); + + /** + * 为每个循环输出创建轻量引用。 + * + * @param resultId 循环结果 ID + * @param iterationCount 已累计迭代数 + * @param outputNames 输出名称 + * @return 输出名称到轻量引用的映射 + */ + default Map references( + String resultId, int iterationCount, List outputNames) { + Map references = new LinkedHashMap<>(); + if (outputNames != null) { + for (String outputName : outputNames) { + references.put( + outputName, + new LoopResultReference(resultId, iterationCount, outputName)); + } + } + return references; + } + + /** + * 解析单个循环输出引用。 + * + * @param reference 循环输出引用 + * @return 与旧实现相同的累计列表 + */ + default Object resolve(LoopResultReference reference) { + return load( + reference.getResultId(), + reference.getIterationCount(), + List.of(reference.getOutputName())) + .get(reference.getOutputName()); + } + + /** + * 递归解析业务输出中的循环引用,供参数读取和 API 边界透明还原。 + * + * @param value 待解析值 + * @return 不包含循环引用的业务值 + */ + default Object resolveReferences(Object value) { + return ReferenceResolver.resolve(this, value); + } + + /** + * 单次递归解析中的批量读取器。 + */ + final class ReferenceResolver { + + private ReferenceResolver() { + } + + /** + * 收集同一循环结果的全部输出名称,并按结果组批量加载一次。 + * + * @param repository 循环结果仓储 + * @param value 待解析值 + * @return 已透明还原的值 + */ + static Object resolve(LoopResultRepository repository, Object value) { + Map> outputNames = + new LinkedHashMap<>(); + java.util.LinkedHashMap inputReferences = + new java.util.LinkedHashMap<>(); + collect(value, outputNames, inputReferences); + Map> loaded = new LinkedHashMap<>(); + outputNames.forEach((key, names) -> loaded.put( + key, + repository.load( + key.resultId, + key.iterationCount, + new java.util.ArrayList<>(names)))); + Map> loadedInputs = + new LinkedHashMap<>(); + inputReferences.forEach((resultId, reference) -> + loadedInputs.put( + resultId, + repository.loadInput(reference))); + return replace(value, loaded, loadedInputs); + } + + /** + * 递归收集循环结果引用。 + * + * @param value 当前值 + * @param outputNames 分组后的输出名称 + */ + private static void collect( + Object value, + Map> outputNames, + Map inputReferences) { + if (value instanceof LoopResultReference) { + LoopResultReference reference = (LoopResultReference) value; + GroupKey key = new GroupKey( + reference.getResultId(), reference.getIterationCount()); + outputNames.computeIfAbsent( + key, ignored -> new java.util.LinkedHashSet<>()) + .add(reference.getOutputName()); + return; + } + if (value instanceof LoopInputReference) { + LoopInputReference reference = + (LoopInputReference) value; + inputReferences.putIfAbsent( + reference.getResultId(), reference); + return; + } + if (value instanceof Map) { + ((Map) value).values().forEach( + item -> collect( + item, outputNames, inputReferences)); + return; + } + if (value instanceof List) { + ((List) value).forEach(item -> collect( + item, outputNames, inputReferences)); + } + } + + /** + * 使用已批量加载的结果递归替换引用。 + * + * @param value 当前值 + * @param loaded 已加载结果 + * @return 替换后的值 + */ + private static Object replace( + Object value, + Map> loaded, + Map> loadedInputs) { + if (value instanceof LoopResultReference) { + LoopResultReference reference = (LoopResultReference) value; + Map outputs = loaded.get(new GroupKey( + reference.getResultId(), reference.getIterationCount())); + return outputs == null ? null : outputs.get(reference.getOutputName()); + } + if (value instanceof LoopInputReference) { + return loadedInputs.get( + ((LoopInputReference) value) + .getResultId()); + } + if (value instanceof Map) { + Map resolved = new LinkedHashMap<>(); + ((Map) value).forEach( + (key, item) -> resolved.put( + key, + replace( + item, + loaded, + loadedInputs))); + return resolved; + } + if (value instanceof List) { + List list = (List) value; + java.util.ArrayList resolved = + new java.util.ArrayList<>(list.size()); + for (Object item : list) { + resolved.add(replace( + item, loaded, loadedInputs)); + } + return resolved; + } + return value; + } + + /** + * 循环结果批量读取分组键。 + */ + private static final class GroupKey { + + private final String resultId; + private final int iterationCount; + + private GroupKey(String resultId, int iterationCount) { + this.resultId = resultId; + this.iterationCount = iterationCount; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof GroupKey)) { + return false; + } + GroupKey that = (GroupKey) other; + return iterationCount == that.iterationCount + && java.util.Objects.equals(resultId, that.resultId); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(resultId, iterationCount); + } + } + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateField.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateField.java index e5d902c..95359ee 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateField.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateField.java @@ -27,5 +27,7 @@ public enum NodeStateField { SUSPEND_NODE_IDS, SUSPEND_FOR_PARAMETERS, EXECUTE_RESULT, - RETRY_COUNT, EXECUTE_COUNT, EXECUTE_EDGE_IDS, LOOP_COUNT, TRIGGER_COUNT, TRIGGER_EDGE_IDS, ENVIRONMENT + RETRY_COUNT, EXECUTE_COUNT, EXECUTE_EDGE_IDS, EXECUTION_ATTEMPT_KEY, + LOOP_COUNT, TRIGGER_COUNT, TRIGGER_EDGE_IDS, ENVIRONMENT, + VERSION } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateRepository.java index 369395d..0f1d779 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateRepository.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/NodeStateRepository.java @@ -21,7 +21,123 @@ import java.util.EnumSet; public interface NodeStateRepository { + /** + * 加载已存在的节点状态。 + * + * @param instanceId 工作流实例 ID + * @param nodeId 节点 ID + * @return 节点状态;纯读取实现可在状态缺失时返回 {@code null},兼容实现可惰性创建 + */ NodeState load(String instanceId, String nodeId); + /** + * 显式创建节点状态。 + * + *

缺省实现兼容旧仓储中由 {@link #load(String, String)} 完成首次创建的行为。 + * 支持持久化或分布式执行的实现应覆盖本方法并原子创建状态。

+ * + * @param instanceId 工作流实例 ID + * @param nodeId 节点 ID + * @param chainStateVersion 创建时关联的工作流状态版本 + * @return 已存在或新创建的节点状态 + */ + default NodeState create(String instanceId, String nodeId, long chainStateVersion) { + NodeState existing = load(instanceId, nodeId); + if (existing != null) { + return existing; + } + NodeState created = new NodeState(); + created.setChainInstanceId(instanceId); + created.setNodeId(nodeId); + if (tryUpdate( + created, + EnumSet.noneOf(NodeStateField.class), + chainStateVersion)) { + return created; + } + return load(instanceId, nodeId); + } + + /** + * 在当前触发器 fencing token 仍有效时显式创建节点状态。 + * + * @param instanceId 工作流实例 ID + * @param nodeId 节点 ID + * @param chainStateVersion 工作流状态版本 + * @param fencingToken 当前触发器 token;非触发器调用为 {@code 0} + * @return 已存在或新创建的节点状态 + */ + default NodeState create( + String instanceId, String nodeId, long chainStateVersion, long fencingToken) { + return create(instanceId, nodeId, chainStateVersion); + } + + /** + * 在实例锁和当前触发器认领租约均有效时显式创建节点状态。 + * + * @param instanceId 工作流实例 ID + * @param nodeId 节点 ID + * @param chainStateVersion 工作流状态版本 + * @param lockFencingToken 当前实例锁 token;本地仓储调用为 {@code 0} + * @param claimId 当前触发器 ID;非触发器调用为 {@code null} + * @param claimGeneration 当前认领代际;非触发器调用为 {@code 0} + * @return 已存在或新创建的节点状态 + */ + default NodeState create( + String instanceId, + String nodeId, + long chainStateVersion, + long lockFencingToken, + String claimId, + long claimGeneration) { + return create(instanceId, nodeId, chainStateVersion, lockFencingToken); + } + + /** + * 按版本尝试提交节点状态。 + * + * @param newState 待提交的新状态 + * @param fields 本次变更字段 + * @param chainStateVersion 本次提交依赖的工作流状态版本 + * @return 提交成功时为 {@code true},版本冲突时为 {@code false} + */ boolean tryUpdate(NodeState newState, EnumSet fields, long chainStateVersion); + + /** + * 在工作流版本和 fencing token 同时有效时提交节点状态。 + * + * @param newState 待提交节点状态 + * @param fields 变化字段 + * @param chainStateVersion 工作流状态版本 + * @param fencingToken 当前触发器 token;非触发器调用为 {@code 0} + * @return 提交成功时为 {@code true} + */ + default boolean tryUpdate( + NodeState newState, + EnumSet fields, + long chainStateVersion, + long fencingToken) { + return tryUpdate(newState, fields, chainStateVersion); + } + + /** + * 在工作流版本、实例锁和当前触发器认领租约同时有效时提交节点状态。 + * + * @param newState 待提交节点状态 + * @param fields 变化字段 + * @param chainStateVersion 工作流状态版本 + * @param lockFencingToken 当前实例锁 token;本地仓储调用为 {@code 0} + * @param claimId 当前触发器 ID;非触发器调用为 {@code null} + * @param claimGeneration 当前认领代际;非触发器调用为 {@code 0} + * @return 提交成功时为 {@code true} + */ + default boolean tryUpdate( + NodeState newState, + EnumSet fields, + long chainStateVersion, + long lockFencingToken, + String claimId, + long claimGeneration) { + return tryUpdate(newState, fields, chainStateVersion, lockFencingToken); + } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java index a63bfa3..130d1a8 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java @@ -25,6 +25,7 @@ import com.easyagents.flow.core.chain.repository.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Serializable; import java.util.*; import java.util.concurrent.*; @@ -39,24 +40,61 @@ import java.util.concurrent.*; public class ChainExecutor { private static final Logger log = LoggerFactory.getLogger(ChainExecutor.class); + private static final String NESTED_DEPTH_MEMORY_KEY = + "__tinyflow.nesting.depth"; + private static final String DEFINITION_CALL_PATH_MEMORY_KEY = + "__tinyflow.nesting.definitionPath"; + /** + * 子工作流触发器使用的独立执行通道。 + */ + public static final String CHILD_WORKFLOW_EXECUTION_LANE_PREFIX = + "child-workflow:"; + private volatile Semaphore rootChildExecutionPermits = + new Semaphore(32, true); + private volatile long persistentOutcomePollMillis = 500L; + private volatile int maxChildExecutionLaneDepth = + Integer.MAX_VALUE; + private static final String CHILD_EXECUTION_REFERENCE_KEY = + "__tinyflow.workflowNode.childExecution"; + /** + * 进程内定义快照仅作为热点缓存;持久快照负责跨节点和跨进程恢复。 + */ + private static final int MAX_ACTIVE_DEFINITIONS = 1024; private final ChainDefinitionRepository definitionRepository; private final ChainStateRepository chainStateRepository; private final NodeStateRepository nodeStateRepository; + private final LoopResultRepository loopResultRepository; + private final ChainDefinitionSnapshotRepository definitionSnapshotRepository; private final TriggerScheduler triggerScheduler; + private final ExecutionBudget executionBudget; private final EventManager eventManager = new EventManager(); /** 等待同步执行结果的任务,按工作流实例 ID 进行常量时间路由。 */ private final ConcurrentMap>> pendingExecutions = new ConcurrentHashMap<>(); + /** + * 活跃工作流实例使用的定义快照,避免每个节点触发都重新加载和解析定义。 + */ + private final Map activeDefinitions = + Collections.synchronizedMap(new LinkedHashMap<>( + MAX_ACTIVE_DEFINITIONS + 1, 0.75F, true) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + return size() > MAX_ACTIVE_DEFINITIONS; + } + }); public ChainExecutor(ChainDefinitionRepository definitionRepository , ChainStateRepository chainStateRepository , NodeStateRepository nodeStateRepository ) { - this.definitionRepository = definitionRepository; - this.chainStateRepository = chainStateRepository; - this.nodeStateRepository = nodeStateRepository; - this.triggerScheduler = ChainRuntime.triggerScheduler(); - registerRuntimeCallbacks(); + this(definitionRepository, + chainStateRepository, + nodeStateRepository, + new InMemoryLoopResultRepository(), + new InMemoryChainDefinitionSnapshotRepository(), + ChainRuntime.triggerScheduler(), + ExecutionBudget.defaults()); } @@ -64,10 +102,92 @@ public class ChainExecutor { , ChainStateRepository chainStateRepository , NodeStateRepository nodeStateRepository , TriggerScheduler triggerScheduler) { + this(definitionRepository, + chainStateRepository, + nodeStateRepository, + new InMemoryLoopResultRepository(), + new InMemoryChainDefinitionSnapshotRepository(), + triggerScheduler, + ExecutionBudget.defaults()); + } + + /** + * 创建使用指定调度器和资源预算的执行器。 + * + * @param definitionRepository 工作流定义仓储 + * @param chainStateRepository 工作流状态仓储 + * @param nodeStateRepository 节点状态仓储 + * @param triggerScheduler 触发调度器 + * @param executionBudget 平台资源保护预算 + */ + public ChainExecutor(ChainDefinitionRepository definitionRepository + , ChainStateRepository chainStateRepository + , NodeStateRepository nodeStateRepository + , TriggerScheduler triggerScheduler + , ExecutionBudget executionBudget) { + this(definitionRepository, + chainStateRepository, + nodeStateRepository, + new InMemoryLoopResultRepository(), + new InMemoryChainDefinitionSnapshotRepository(), + triggerScheduler, + executionBudget); + } + + /** + * 创建使用指定调度器、循环结果仓储和资源预算的执行器。 + * + * @param definitionRepository 工作流定义仓储 + * @param chainStateRepository 工作流状态仓储 + * @param nodeStateRepository 节点状态仓储 + * @param loopResultRepository 循环累计结果仓储 + * @param triggerScheduler 触发调度器 + * @param executionBudget 平台资源保护预算 + */ + public ChainExecutor(ChainDefinitionRepository definitionRepository + , ChainStateRepository chainStateRepository + , NodeStateRepository nodeStateRepository + , LoopResultRepository loopResultRepository + , TriggerScheduler triggerScheduler + , ExecutionBudget executionBudget) { + this(definitionRepository, + chainStateRepository, + nodeStateRepository, + loopResultRepository, + new InMemoryChainDefinitionSnapshotRepository(), + triggerScheduler, + executionBudget); + } + + /** + * 创建使用持久定义快照的执行器。 + * + * @param definitionRepository 工作流定义仓储 + * @param chainStateRepository 工作流状态仓储 + * @param nodeStateRepository 节点状态仓储 + * @param loopResultRepository 循环累计结果仓储 + * @param definitionSnapshotRepository 实例级定义快照仓储 + * @param triggerScheduler 触发调度器 + * @param executionBudget 平台资源保护预算 + */ + public ChainExecutor(ChainDefinitionRepository definitionRepository + , ChainStateRepository chainStateRepository + , NodeStateRepository nodeStateRepository + , LoopResultRepository loopResultRepository + , ChainDefinitionSnapshotRepository definitionSnapshotRepository + , TriggerScheduler triggerScheduler + , ExecutionBudget executionBudget) { this.definitionRepository = definitionRepository; this.chainStateRepository = chainStateRepository; this.nodeStateRepository = nodeStateRepository; + this.loopResultRepository = loopResultRepository == null + ? new InMemoryLoopResultRepository() + : loopResultRepository; + this.definitionSnapshotRepository = definitionSnapshotRepository == null + ? new InMemoryChainDefinitionSnapshotRepository() + : definitionSnapshotRepository; this.triggerScheduler = triggerScheduler; + this.executionBudget = executionBudget == null ? ExecutionBudget.defaults() : executionBudget; registerRuntimeCallbacks(); } @@ -80,33 +200,47 @@ public class ChainExecutor { public Map execute(String definitionId, Map variables, long timeout, TimeUnit unit) { Chain chain = createChain(definitionId); String stateInstanceId = chain.getStateInstanceId(); - CompletableFuture> future = new CompletableFuture<>(); - - CompletableFuture> existing = pendingExecutions.putIfAbsent(stateInstanceId, future); - if (existing != null) { - throw new IllegalStateException("Duplicate pending chain execution: " + stateInstanceId); - } try { chain.start(variables); - Map result = future.get(timeout, unit); + Map result = awaitPersistentOutcome( + stateInstanceId, timeout, unit, null); clearDefaultStates(result); return result; } catch (TimeoutException e) { - future.cancel(true); + cancel(stateInstanceId, "Execution timed out"); throw new RuntimeException("Execution timed out", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - future.cancel(true); + cancel(stateInstanceId, "Execution interrupted"); throw new RuntimeException("Execution interrupted", e); } catch (Throwable e) { - future.cancel(true); - throw new RuntimeException("Execution failed", e.getCause()); + throw new RuntimeException("Execution failed", e); } finally { - pendingExecutions.remove(stateInstanceId, future); + activeDefinitions.remove(stateInstanceId); } } + /** + * 取消仍在运行的工作流实例。 + * + *

取消状态写入后,未开始的触发器会在执行入口短路;已经完成的外部 I/O 也会在提交 + * 结果前重新检查状态,避免继续推进下游。

+ * + * @param stateInstanceId 工作流实例 ID + * @param message 取消原因 + * @return 本次是否完成了非终态到取消状态的转换 + */ + public boolean cancel(String stateInstanceId, String message) { + ChainState state = chainStateRepository.load(stateInstanceId); + if (state == null) { + return false; + } + ChainDefinition definition = getDefinitionForInstance(state); + Chain chain = configureChain(definition, stateInstanceId); + return chain.cancel(message); + } + /** * 注册工作流调度和同步结果路由回调。 */ @@ -114,6 +248,79 @@ public class ChainExecutor { eventManager.addEventListener(ChainStatusChangeEvent.class, this::completePendingExecution); eventManager.addChainErrorListener(this::failPendingExecution); triggerScheduler.registerConsumer(this::accept); + triggerScheduler.registerFailureListener( + this::failDeadLetteredTrigger); + } + + /** + * 配置同步子工作流的容量和持久状态轮询参数。 + * + *

该方法应在执行器对外提供服务前调用。

+ * + * @param rootPermits 根级同步子流程最大并发 + * @param pollMillis 持久终态轮询间隔毫秒数 + * @param laneMaxDepth 已注册独立执行通道覆盖的最大深度 + */ + public void configureChildWorkflowRuntime( + int rootPermits, + long pollMillis, + int laneMaxDepth) { + if (rootPermits <= 0 || pollMillis <= 0L + || laneMaxDepth <= 0) { + throw new IllegalArgumentException( + "Child workflow runtime values must be positive"); + } + this.rootChildExecutionPermits = + new Semaphore(rootPermits, true); + this.persistentOutcomePollMillis = pollMillis; + this.maxChildExecutionLaneDepth = laneMaxDepth; + } + + /** + * 将已成功写入死信的触发器对应实例收敛为失败终态。 + * + *

状态更新使用实例锁 fencing token,避免旧节点在锁失效后覆盖新 owner 的 + * 业务终态;终态实例保持原状态。

+ * + * @param trigger 已死信触发器 + * @param failure 最后一次失败 + */ + private boolean failDeadLetteredTrigger( + Trigger trigger, Throwable failure) { + if (trigger == null + || trigger.getStateInstanceId() == null) { + return true; + } + String instanceId = trigger.getStateInstanceId(); + Throwable cause = failure == null + ? new ChainException( + "Workflow trigger delivery attempts exhausted: " + + trigger.getId()) + : failure; + try { + ChainState state = + chainStateRepository.load(instanceId); + if (state == null + || (state.getStatus() != null + && state.getStatus().isTerminal())) { + return true; + } + ChainDefinition definition = + getDefinitionForInstance(state); + if (definition == null) { + definition = new ChainDefinition(); + definition.setId(state.getChainDefinitionId()); + } + Chain chain = configureChain( + definition, instanceId); + return chain.failTerminal(cause); + } catch (Throwable terminalError) { + log.error( + "Failed to mark dead-lettered workflow terminal, " + + "instanceId={}, triggerId={}", + instanceId, trigger.getId(), terminalError); + return false; + } } /** @@ -130,19 +337,28 @@ public class ChainExecutor { String stateInstanceId = chain.getStateInstanceId(); CompletableFuture> future = pendingExecutions.get(stateInstanceId); - if (future == null) { - return; - } - try { - ChainState state = chainStateRepository.load(stateInstanceId); - if (state == null) { - throw new ChainException("Chain state not found: " + stateInstanceId); + if (future != null) { + ChainState state = chainStateRepository.load(stateInstanceId); + if (state == null) { + throw new ChainException("Chain state not found: " + stateInstanceId); + } + @SuppressWarnings("unchecked") + Map execResult = (Map) + loopResultRepository.resolveReferences(state.getExecuteResult()); + future.complete(execResult != null ? execResult : Collections.emptyMap()); } - Map execResult = state.getExecuteResult(); - future.complete(execResult != null ? execResult : Collections.emptyMap()); } catch (Exception error) { - future.completeExceptionally(error); + if (future != null) { + future.completeExceptionally(error); + } + log.error( + "Failed to complete workflow execution continuation, instanceId={}", + stateInstanceId, + error); + } finally { + activeDefinitions.remove(stateInstanceId); + definitionSnapshotRepository.remove(stateInstanceId); } } @@ -153,10 +369,16 @@ public class ChainExecutor { * @param chain 发生异常的工作流实例 */ private void failPendingExecution(Throwable error, Chain chain) { - CompletableFuture> future = pendingExecutions.get(chain.getStateInstanceId()); + String stateInstanceId = chain.getStateInstanceId(); + CompletableFuture> future = pendingExecutions.get(stateInstanceId); if (future != null) { future.completeExceptionally(error); } + activeDefinitions.remove(stateInstanceId); + ChainState state = chainStateRepository.load(stateInstanceId); + if (state != null && state.getStatus().isTerminal()) { + definitionSnapshotRepository.remove(stateInstanceId); + } } /** @@ -176,8 +398,416 @@ public class ChainExecutor { public String executeAsync(String definitionId, Map variables) { Chain chain = createChain(definitionId); - chain.start(variables); - return chain.getStateInstanceId(); + try { + chain.start(variables); + return chain.getStateInstanceId(); + } catch (RuntimeException | Error error) { + activeDefinitions.remove(chain.getStateInstanceId()); + definitionSnapshotRepository.remove(chain.getStateInstanceId()); + throw error; + } + } + + /** + * 在独立触发执行通道中同步执行子工作流。 + * + *

调用者继续获得与历史实现一致的同步结果;子流程自身的节点触发器在独立 + * worker lane 执行,因此父节点等待不会占满子流程所需的普通节点工作线程。 + * 根级子流程并发受宽松许可保护,防止异常调用一次创建过多等待线程。

+ * + * @param definitionId 子流程定义 ID + * @param variables 子流程输入 + * @param parentChain 父流程 + * @param parentNodeId 父工作流节点 ID + * @return 子流程输出 + */ + public Map executeChild( + String definitionId, + Map variables, + Chain parentChain, + String parentNodeId) { + Objects.requireNonNull(parentChain, "parentChain required"); + if (parentNodeId == null || parentNodeId.isBlank()) { + throw new IllegalArgumentException("parentNodeId required"); + } + ChainState parentState = chainStateRepository.load( + parentChain.getStateInstanceId()); + int parentDepth = readNestedDepth(parentState); + int childDepth = parentDepth + 1; + executionBudget.checkNestedDepth(parentNodeId, childDepth); + if (childDepth > maxChildExecutionLaneDepth) { + throw new ExecutionBudgetExceededException( + "Child workflow depth " + + childDepth + + " exceeds registered execution lanes " + + maxChildExecutionLaneDepth); + } + List callPath = readDefinitionCallPath( + parentState, parentChain.getDefinition().getId()); + String canonicalChildId = canonicalDefinitionId(definitionId); + if (callPath.contains(canonicalChildId)) { + throw new ExecutionBudgetExceededException( + "Recursive workflow call detected at " + + parentNodeId + + ": " + + String.join(" -> ", callPath) + + " -> " + + canonicalChildId); + } + List childCallPath = new ArrayList<>(callPath); + childCallPath.add(canonicalChildId); + Trigger currentTrigger = TriggerContext.getCurrentTrigger(); + String invocationId = currentTrigger == null + ? "direct:" + UUID.randomUUID() + : (currentTrigger.getLogicalExecutionId() == null + || currentTrigger.getLogicalExecutionId().isBlank() + ? currentTrigger.getId() + : currentTrigger.getLogicalExecutionId()); + boolean rootPermit = false; + try { + if (parentDepth == 0) { + if (!rootChildExecutionPermits.tryAcquire( + 1L, TimeUnit.SECONDS)) { + throw new RetryableTriggerException( + "子工作流并发繁忙,请稍后重试", null); + } + rootPermit = true; + } + ChildExecutionReference reference = + prepareChildExecution( + definitionId, + parentChain, + parentNodeId, + invocationId, + childDepth, + childCallPath); + startChildIfReady(reference, variables); + Map result = awaitPersistentOutcome( + reference.childInstanceId(), + Long.MAX_VALUE, + TimeUnit.SECONDS, + parentChain); + markChildExecutionCompleted( + parentChain, parentNodeId, reference); + clearDefaultStates(result); + return result; + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new RuntimeException( + "Child workflow execution interrupted", error); + } catch (TimeoutException impossible) { + throw new IllegalStateException( + "Unexpected child workflow timeout", impossible); + } finally { + if (rootPermit) { + rootChildExecutionPermits.release(); + } + } + } + + /** + * 在父实例短锁内复用或创建持久子流程关联。 + * + * @param definitionId 子流程定义 ID + * @param parentChain 父流程 + * @param parentNodeId 父节点 ID + * @param invocationId 父节点逻辑执行 ID + * @param childDepth 子流程综合深度 + * @param childCallPath 子流程调用链 + * @return 持久子流程关联 + */ + private ChildExecutionReference prepareChildExecution( + String definitionId, + Chain parentChain, + String parentNodeId, + String invocationId, + int childDepth, + List childCallPath) { + return parentChain.executeWithLock( + parentChain.getStateInstanceId(), + 10L, + TimeUnit.SECONDS, + () -> { + NodeState nodeState = + parentChain.updateNodeStateSafely( + parentNodeId, state -> null); + Object existingValue = nodeState.getMemory().get( + CHILD_EXECUTION_REFERENCE_KEY); + if (existingValue instanceof ChildExecutionReference) { + ChildExecutionReference existing = + (ChildExecutionReference) existingValue; + if (Objects.equals( + existing.invocationId(), invocationId)) { + return existing; + } + ChainState existingChild = chainStateRepository.load( + existing.childInstanceId()); + if (existingChild != null + && (existingChild.getStatus() == null + || !existingChild.getStatus().isTerminal())) { + throw new RetryableTriggerException( + "前一次子工作流仍在执行", null); + } + } + Chain child = createChain(definitionId); + child.updateStateSafely(state -> { + state.getMemory().put( + NESTED_DEPTH_MEMORY_KEY, childDepth); + state.getMemory().put( + DEFINITION_CALL_PATH_MEMORY_KEY, + new ArrayList<>(childCallPath)); + return EnumSet.of(ChainStateField.MEMORY); + }); + ChildExecutionReference created = + new ChildExecutionReference( + invocationId, + child.getStateInstanceId(), + childDepth, + false); + parentChain.updateNodeStateSafely( + parentNodeId, + state -> { + state.getMemory().put( + CHILD_EXECUTION_REFERENCE_KEY, + created); + return EnumSet.of(NodeStateField.MEMORY); + }); + return created; + }); + } + + /** + * 在子实例仍为 READY 时幂等启动。 + * + * @param reference 子流程关联 + * @param variables 子流程输入 + */ + private void startChildIfReady( + ChildExecutionReference reference, + Map variables) { + ChainState childState = chainStateRepository.load( + reference.childInstanceId()); + if (childState == null) { + throw new ChainException( + "Child chain state not found: " + + reference.childInstanceId()); + } + ChainDefinition definition = + getDefinitionForInstance(childState); + if (definition == null) { + throw new ChainException( + "Child chain definition not found: " + + reference.childInstanceId()); + } + Chain child = configureChain( + definition, reference.childInstanceId()); + child.setExecutionLane(childExecutionLane(reference.depth())); + child.setNestedDepthBase(reference.depth()); + child.executeWithLock( + reference.childInstanceId(), + 10L, + TimeUnit.SECONDS, + () -> { + ChainState latest = chainStateRepository.load( + reference.childInstanceId()); + if (latest != null + && (latest.getStatus() == ChainStatus.READY + || latest.getStatus() + == ChainStatus.RUNNING)) { + child.start(variables); + } + return null; + }); + } + + /** + * 标记当前父节点关联已观察到子流程终态。 + * + * @param parentChain 父流程 + * @param parentNodeId 父节点 ID + * @param reference 子流程关联 + */ + private void markChildExecutionCompleted( + Chain parentChain, + String parentNodeId, + ChildExecutionReference reference) { + parentChain.updateNodeStateSafely(parentNodeId, state -> { + Object current = state.getMemory().get( + CHILD_EXECUTION_REFERENCE_KEY); + if (!(current instanceof ChildExecutionReference) + || !Objects.equals( + ((ChildExecutionReference) current).invocationId(), + reference.invocationId())) { + return null; + } + state.getMemory().put( + CHILD_EXECUTION_REFERENCE_KEY, + new ChildExecutionReference( + reference.invocationId(), + reference.childInstanceId(), + reference.depth(), + true)); + return EnumSet.of(NodeStateField.MEMORY); + }); + } + + /** + * 按综合嵌套深度生成独立子流程通道。 + * + * @param depth 综合嵌套深度 + * @return 通道名 + */ + public static String childExecutionLane(int depth) { + return CHILD_WORKFLOW_EXECUTION_LANE_PREFIX + + Math.max(1, depth); + } + + /** + * 读取实例持久化的子工作流嵌套深度。 + * + * @param state 工作流状态 + * @return 非负嵌套深度 + */ + private int readNestedDepth(ChainState state) { + if (state == null) { + return 0; + } + Object value = state.getMemory().get(NESTED_DEPTH_MEMORY_KEY); + return value instanceof Number + ? Math.max(0, ((Number) value).intValue()) + : 0; + } + + /** + * 读取并规范化实例的工作流定义调用链。 + * + * @param state 工作流状态 + * @param currentDefinitionId 当前定义 ID + * @return 从根定义到当前定义的调用链 + */ + private List readDefinitionCallPath( + ChainState state, String currentDefinitionId) { + List path = new ArrayList<>(); + Object value = state == null + ? null + : state.getMemory().get(DEFINITION_CALL_PATH_MEMORY_KEY); + if (value instanceof Collection) { + for (Object item : (Collection) value) { + if (item != null) { + path.add(canonicalDefinitionId(String.valueOf(item))); + } + } + } + if (path.isEmpty() && currentDefinitionId != null) { + path.add(canonicalDefinitionId(currentDefinitionId)); + } + return path; + } + + /** + * 将发布态和草稿态的同一工作流规范化为统一调用身份。 + * + * @param definitionId 定义 ID + * @return 规范化定义 ID + */ + private String canonicalDefinitionId(String definitionId) { + if (definitionId == null) { + return ""; + } + String normalized = definitionId.trim(); + return normalized.startsWith("published:") + ? normalized.substring("published:".length()) + : normalized; + } + + /** + * 轮询持久状态等待工作流终态,允许任意集群实例执行实际触发器。 + * + * @param stateInstanceId 工作流实例 ID + * @param timeout 超时数值 + * @param unit 超时单位 + * @return 已解析业务结果 + * @throws InterruptedException 等待线程被中断 + * @throws TimeoutException 超时 + */ + @SuppressWarnings("unchecked") + private Map awaitPersistentOutcome( + String stateInstanceId, + long timeout, + TimeUnit unit, + Chain parentChain) + throws InterruptedException, TimeoutException { + Objects.requireNonNull(unit, "time unit required"); + long timeoutNanos = timeout == Long.MAX_VALUE + ? Long.MAX_VALUE + : Math.max(0L, unit.toNanos(timeout)); + long startedAt = System.nanoTime(); + while (true) { + if (parentChain != null) { + ChainState parentState = chainStateRepository.load( + parentChain.getStateInstanceId()); + if (parentState == null + || (parentState.getStatus() != null + && parentState.getStatus().isTerminal())) { + cancel( + stateInstanceId, + "Parent workflow is no longer running"); + throw new ChainException( + "Parent workflow ended while child was running"); + } + Trigger owner = TriggerContext.getCurrentTrigger(); + if (owner != null) { + // 认领丢失只终止旧 owner 的等待;durable child 留给新 owner 复用。 + triggerScheduler.assertClaimOwned(owner); + } + } + ChainState state = chainStateRepository.load(stateInstanceId); + if (state == null) { + throw new ChainException( + "Chain state not found: " + stateInstanceId); + } + ChainStatus status = state.getStatus(); + if (status != null && status.isTerminal()) { + if (!status.isSuccess()) { + ExceptionSummary error = state.getError(); + throw new ChainException( + error == null + ? "Workflow ended with status " + status + : error.getMessage()); + } + Map result = + (Map) + loopResultRepository.resolveReferences( + state.getExecuteResult()); + return result == null + ? Collections.emptyMap() + : result; + } + if (timeoutNanos != Long.MAX_VALUE + && System.nanoTime() - startedAt >= timeoutNanos) { + throw new TimeoutException( + "Workflow execution timed out: " + + stateInstanceId); + } + Thread.sleep(persistentOutcomePollMillis); + } + } + + /** + * 父工作流节点与子实例之间的持久关联。 + * + * @param invocationId 父节点逻辑执行 ID + * @param childInstanceId 子流程实例 ID + * @param depth 子流程综合嵌套深度 + * @param completed 是否已观察到终态 + */ + private record ChildExecutionReference( + String invocationId, + String childInstanceId, + int depth, + boolean completed) implements Serializable { + + private static final long serialVersionUID = 1L; } @@ -192,15 +822,21 @@ public class ChainExecutor { public Map executeNode(String definitionId, String nodeId, Map variables) { ChainDefinition chainDefinitionById = definitionRepository.getChainDefinitionById(definitionId); Node node = chainDefinitionById.getNodeById(nodeId); - Chain temp = createChain(definitionId); - if (variables != null && !variables.isEmpty()) { - temp.updateStateSafely(s -> { - s.getMemory().putAll(variables); - temp.applyStartParameterAliases(s.getMemory(), variables); - return EnumSet.of(ChainStateField.MEMORY); - }); + Chain temp = createChain(chainDefinitionById); + try { + temp.initializeState(); + if (variables != null && !variables.isEmpty()) { + temp.updateStateSafely(s -> { + s.getMemory().putAll(variables); + temp.applyStartParameterAliases(s.getMemory(), variables); + return EnumSet.of(ChainStateField.MEMORY); + }); + } + return node.execute(temp); + } finally { + activeDefinitions.remove(temp.getStateInstanceId()); + definitionSnapshotRepository.remove(temp.getStateInstanceId()); } - return node.execute(temp); } @@ -229,17 +865,12 @@ public class ChainExecutor { return; } - ChainDefinition definition = definitionRepository.getChainDefinitionById(state.getChainDefinitionId()); + ChainDefinition definition = getDefinitionForInstance(state); if (definition == null) { return; } - Chain chain = new Chain(definition, state.getInstanceId()); - chain.setTriggerScheduler(triggerScheduler); - chain.setChainStateRepository(chainStateRepository); - chain.setNodeStateRepository(nodeStateRepository); - chain.setEventManager(eventManager); - + Chain chain = configureChain(definition, state.getInstanceId()); chain.resume(variables); } @@ -249,44 +880,132 @@ public class ChainExecutor { if (definition == null) { throw new RuntimeException("Chain definition not found"); } + return createChain(definition); + } + /** + * 使用已加载的定义创建工作流实例,避免同一次调用重复读取定义。 + * + * @param definition 已加载的工作流定义 + * @return 已完成运行时依赖配置的工作流实例 + */ + private Chain createChain(ChainDefinition definition) { String stateInstanceId = UUID.randomUUID().toString(); + activeDefinitions.put(stateInstanceId, definition); + try { + definitionSnapshotRepository.save(stateInstanceId, definition); + Chain chain = configureChain(definition, stateInstanceId); + chain.initializeState(); + return chain; + } catch (RuntimeException | Error error) { + activeDefinitions.remove(stateInstanceId); + try { + definitionSnapshotRepository.remove(stateInstanceId); + } catch (RuntimeException cleanupError) { + error.addSuppressed(cleanupError); + } + throw error; + } + } + + /** + * 为工作流实例配置共享运行时依赖。 + * + * @param definition 工作流定义 + * @param stateInstanceId 状态实例 ID + * @return 配置完成的工作流实例 + */ + private Chain configureChain(ChainDefinition definition, String stateInstanceId) { + return configureChain(definition, stateInstanceId, null); + } + + /** + * 为工作流实例配置共享运行时依赖,并复用调用方已经加载的状态。 + * + * @param definition 工作流定义 + * @param stateInstanceId 状态实例 ID + * @param persistedState 已加载状态;为 {@code null} 时按需读取 + * @return 配置完成的工作流实例 + */ + private Chain configureChain( + ChainDefinition definition, + String stateInstanceId, + ChainState persistedState) { Chain chain = new Chain(definition, stateInstanceId); chain.setTriggerScheduler(triggerScheduler); chain.setChainStateRepository(chainStateRepository); chain.setNodeStateRepository(nodeStateRepository); + chain.setLoopResultRepository(loopResultRepository); chain.setEventManager(eventManager); - + chain.setExecutionBudget(executionBudget); + ChainState state = persistedState == null + ? chainStateRepository.load(stateInstanceId) + : persistedState; + int nestedDepth = readNestedDepth(state); + chain.setNestedDepthBase(nestedDepth); + if (nestedDepth > 0) { + chain.setExecutionLane(childExecutionLane(nestedDepth)); + } return chain; } + /** + * 获取工作流实例启动时使用的定义快照。 + * + * @param state 工作流状态 + * @return 活跃定义快照;当前实例首次由本节点接管时从仓储加载 + */ + private ChainDefinition getDefinitionForInstance(ChainState state) { + String stateInstanceId = state.getInstanceId(); + ChainDefinition definition = activeDefinitions.get(stateInstanceId); + if (definition != null) { + return definition; + } + ChainDefinition loaded = definitionSnapshotRepository.load(stateInstanceId); + if (loaded == null) { + // 兼容升级前已经启动、尚未持久化定义快照的实例。 + loaded = definitionRepository.getChainDefinitionById(state.getChainDefinitionId()); + } + if (loaded == null) { + return null; + } + synchronized (activeDefinitions) { + ChainDefinition existing = activeDefinitions.get(stateInstanceId); + if (existing != null) { + return existing; + } + activeDefinitions.put(stateInstanceId, loaded); + return loaded; + } + } + private void accept(Trigger trigger, ExecutorService worker) { ChainState state = chainStateRepository.load(trigger.getStateInstanceId()); if (state == null) { - throw new ChainException("Chain state not found"); + // 状态已过期或被清理时,该触发器已经失去业务目标,直接确认避免无限热重放。 + return; } - ChainDefinition definition = definitionRepository.getChainDefinitionById(state.getChainDefinitionId()); + ChainDefinition definition = getDefinitionForInstance(state); if (definition == null) { - throw new ChainException("Chain definition not found"); + throw new NonRetryableTriggerException( + "Chain definition not found: " + state.getChainDefinitionId()); } - Chain chain = new Chain(definition, trigger.getStateInstanceId()); - chain.setTriggerScheduler(triggerScheduler); - chain.setChainStateRepository(chainStateRepository); - chain.setNodeStateRepository(nodeStateRepository); - chain.setEventManager(eventManager); + Chain chain = configureChain( + definition, trigger.getStateInstanceId(), state); String nodeId = trigger.getNodeId(); if (nodeId == null) { - throw new ChainException("Node ID not found in trigger."); + throw new NonRetryableTriggerException("Node ID not found in trigger."); } Node node = definition.getNodeById(nodeId); if (node == null) { - throw new ChainException("Node not found in definition(id: " + definition.getId() + ")"); + throw new NonRetryableTriggerException( + "Node not found in definition(id: " + definition.getId() + ")"); } chain.executeNode(node, trigger); @@ -345,6 +1064,16 @@ public class ChainExecutor { return triggerScheduler; } + /** + * 在查询/API 边界透明还原循环结果引用。 + * + * @param value 可能包含引用的值 + * @return 业务可见值 + */ + public Object resolveResultReferences(Object value) { + return loopResultRepository.resolveReferences(value); + } + public EventManager getEventManager() { return eventManager; } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudget.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudget.java new file mode 100644 index 0000000..a48042e --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudget.java @@ -0,0 +1,188 @@ +package com.easyagents.flow.core.chain.runtime; + +import java.io.Serializable; + +/** + * 工作流执行的全局资源保护预算。 + *

+ * 所有默认值均为宽松的失控保护值。小于等于 {@code 0} 的配置表示关闭对应保护, + * 节点自身的循环次数、退出条件和重试配置仍按原有语义优先生效。 + */ +public final class ExecutionBudget implements Serializable { + private static final long serialVersionUID = 1L; + + + public static final long DEFAULT_MAX_ITERATIONS = 100_000L; + /** + * 缺省不限制墙钟时长,避免人工确认或长期挂起时间被误计为执行耗时。 + */ + public static final long DEFAULT_MAX_DURATION_MILLIS = 0L; + public static final long DEFAULT_MAX_CHILD_EXECUTIONS = 1_000_000L; + public static final long DEFAULT_MAX_ACCUMULATED_BYTES = 512L * 1024L * 1024L; + public static final int DEFAULT_MAX_NESTED_DEPTH = 32; + /** + * 热状态硬限制缺省关闭。循环历史已通过引用隔离,开启限制时由部署方按实际负载设置, + * 避免估算误差改变既有业务语义。 + */ + public static final long DEFAULT_MAX_HOT_STATE_BYTES = 0L; + + private final long maxIterations; + private final long maxDurationMillis; + private final long maxChildExecutions; + private final long maxAccumulatedBytes; + private final int maxNestedDepth; + private final long maxHotStateBytes; + + /** + * 创建执行预算。 + * + * @param maxIterations 单循环最大迭代次数 + * @param maxDurationMillis 单实例最大运行毫秒数 + * @param maxChildExecutions 单实例最大节点执行次数 + * @param maxAccumulatedBytes 单循环最大累计结果字节数 + * @param maxNestedDepth 最大循环嵌套深度 + * @param maxHotStateBytes 单实例热状态建议最大字节数 + */ + public ExecutionBudget(long maxIterations, + long maxDurationMillis, + long maxChildExecutions, + long maxAccumulatedBytes, + int maxNestedDepth, + long maxHotStateBytes) { + this.maxIterations = maxIterations; + this.maxDurationMillis = maxDurationMillis; + this.maxChildExecutions = maxChildExecutions; + this.maxAccumulatedBytes = maxAccumulatedBytes; + this.maxNestedDepth = maxNestedDepth; + this.maxHotStateBytes = maxHotStateBytes; + } + + /** + * 创建使用宽松缺省值的执行预算。 + * + * @return 默认执行预算 + */ + public static ExecutionBudget defaults() { + return new ExecutionBudget( + DEFAULT_MAX_ITERATIONS, + DEFAULT_MAX_DURATION_MILLIS, + DEFAULT_MAX_CHILD_EXECUTIONS, + DEFAULT_MAX_ACCUMULATED_BYTES, + DEFAULT_MAX_NESTED_DEPTH, + DEFAULT_MAX_HOT_STATE_BYTES); + } + + public long getMaxIterations() { + return maxIterations; + } + + public long getMaxDurationMillis() { + return maxDurationMillis; + } + + public long getMaxChildExecutions() { + return maxChildExecutions; + } + + public long getMaxAccumulatedBytes() { + return maxAccumulatedBytes; + } + + public int getMaxNestedDepth() { + return maxNestedDepth; + } + + public long getMaxHotStateBytes() { + return maxHotStateBytes; + } + + /** + * 校验循环迭代总数。 + * + * @param nodeId 循环节点 ID + * @param iterations 计划迭代次数 + * @throws ExecutionBudgetExceededException 超过启用的迭代预算时抛出 + */ + public void checkIterations(String nodeId, long iterations) { + if (maxIterations > 0 && iterations > maxIterations) { + throw new ExecutionBudgetExceededException( + "Loop iteration budget exceeded for node " + nodeId + + ": " + iterations + " > " + maxIterations); + } + } + + /** + * 校验循环嵌套深度。 + * + * @param nodeId 节点 ID + * @param depth 当前深度 + * @throws ExecutionBudgetExceededException 超过启用的深度预算时抛出 + */ + public void checkNestedDepth(String nodeId, int depth) { + if (maxNestedDepth > 0 && depth > maxNestedDepth) { + throw new ExecutionBudgetExceededException( + "Loop nested depth budget exceeded for node " + nodeId + + ": " + depth + " > " + maxNestedDepth); + } + } + + /** + * 校验循环累计结果大小。 + * + * @param nodeId 循环节点 ID + * @param accumulatedBytes 当前累计估算字节数 + * @throws ExecutionBudgetExceededException 超过启用的累计结果预算时抛出 + */ + public void checkAccumulatedBytes(String nodeId, long accumulatedBytes) { + if (maxAccumulatedBytes > 0 && accumulatedBytes > maxAccumulatedBytes) { + throw new ExecutionBudgetExceededException( + "Loop accumulated result budget exceeded for node " + nodeId + + ": " + accumulatedBytes + " > " + maxAccumulatedBytes); + } + } + + /** + * 校验单实例节点执行次数。 + * + * @param executions 当前节点执行次数 + * @throws ExecutionBudgetExceededException 超过启用的执行预算时抛出 + */ + public void checkChildExecutions(long executions) { + if (maxChildExecutions > 0 && executions > maxChildExecutions) { + throw new ExecutionBudgetExceededException( + "Workflow child execution budget exceeded: " + + executions + " > " + maxChildExecutions); + } + } + + /** + * 校验单实例运行时长。 + * + * @param startedAtMillis 实例开始时间 + * @param nowMillis 当前时间 + * @throws ExecutionBudgetExceededException 超过启用的时长预算时抛出 + */ + public void checkDuration(long startedAtMillis, long nowMillis) { + if (maxDurationMillis > 0 + && startedAtMillis > 0 + && nowMillis - startedAtMillis > maxDurationMillis) { + throw new ExecutionBudgetExceededException( + "Workflow duration budget exceeded: " + + (nowMillis - startedAtMillis) + "ms > " + maxDurationMillis + "ms"); + } + } + + /** + * 校验工作流热状态估算大小。 + * + * @param estimatedBytes 当前热状态估算字节数 + * @throws ExecutionBudgetExceededException 超过启用的热状态预算时抛出 + */ + public void checkHotStateBytes(long estimatedBytes) { + if (maxHotStateBytes > 0 && estimatedBytes > maxHotStateBytes) { + throw new ExecutionBudgetExceededException( + "Workflow hot state budget exceeded: " + + estimatedBytes + " > " + maxHotStateBytes); + } + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudgetExceededException.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudgetExceededException.java new file mode 100644 index 0000000..44d5ef7 --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ExecutionBudgetExceededException.java @@ -0,0 +1,18 @@ +package com.easyagents.flow.core.chain.runtime; + +import com.easyagents.flow.core.chain.ChainException; + +/** + * 工作流实例超过平台资源保护预算时抛出的异常。 + */ +public class ExecutionBudgetExceededException extends ChainException { + + /** + * 创建预算超限异常。 + * + * @param message 可审计的超限原因 + */ + public ExecutionBudgetExceededException(String message) { + super(message); + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/InMemoryTriggerStore.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/InMemoryTriggerStore.java index 550b4d6..c7634f9 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/InMemoryTriggerStore.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/InMemoryTriggerStore.java @@ -17,13 +17,16 @@ package com.easyagents.flow.core.chain.runtime; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; public class InMemoryTriggerStore implements TriggerStore { private final ConcurrentHashMap store = new ConcurrentHashMap<>(); + private final ConcurrentHashMap fencingTokens = new ConcurrentHashMap<>(); @Override public Trigger save(Trigger trigger) { @@ -34,6 +37,18 @@ public class InMemoryTriggerStore implements TriggerStore { return trigger; } + /** + * {@inheritDoc} + */ + @Override + public boolean saveIfAbsent(Trigger trigger) { + if (trigger.getId() == null || trigger.getId().isBlank()) { + throw new IllegalArgumentException("Stable trigger ID required"); + } + return store.putIfAbsent( + trigger.getId(), trigger) == null; + } + @Override public boolean remove(String triggerId) { return store.remove(triggerId) != null; @@ -46,12 +61,35 @@ public class InMemoryTriggerStore implements TriggerStore { @Override public List findDue(long uptoTimestamp) { - return null; + List due = new ArrayList<>(); + for (Trigger trigger : store.values()) { + if (trigger.getTriggerAt() <= uptoTimestamp) { + due.add(trigger); + } + } + due.sort(Comparator.comparingLong(Trigger::getTriggerAt)); + return due; } @Override public List findAllPending() { return new ArrayList<>(store.values()); } -} + /** + * {@inheritDoc} + */ + @Override + public Trigger claim(String triggerId, long leaseMillis) { + Trigger trigger = store.remove(triggerId); + if (trigger != null) { + String fencingScope = trigger.getStateInstanceId() == null + ? "__trigger__:" + trigger.getId() + : trigger.getStateInstanceId(); + trigger.setFencingToken(fencingTokens + .computeIfAbsent(fencingScope, ignored -> new AtomicLong()) + .incrementAndGet()); + } + return trigger; + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/NonRetryableTriggerException.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/NonRetryableTriggerException.java new file mode 100644 index 0000000..2fab07d --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/NonRetryableTriggerException.java @@ -0,0 +1,16 @@ +package com.easyagents.flow.core.chain.runtime; + +/** + * 表示触发器内容已经无法继续执行,应进入死信而非无限重放。 + */ +public class NonRetryableTriggerException extends RuntimeException { + + /** + * 创建不可重试触发器异常。 + * + * @param message 异常说明 + */ + public NonRetryableTriggerException(String message) { + super(message); + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/RetryableTriggerException.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/RetryableTriggerException.java new file mode 100644 index 0000000..b792710 --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/RetryableTriggerException.java @@ -0,0 +1,17 @@ +package com.easyagents.flow.core.chain.runtime; + +/** + * 表示当前节点遇到短暂基础设施冲突,应重新投递同一触发器且不消耗业务重试次数。 + */ +public class RetryableTriggerException extends RuntimeException { + + /** + * 创建可重新投递异常。 + * + * @param message 异常说明 + * @param cause 原始异常 + */ + public RetryableTriggerException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/Trigger.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/Trigger.java index 702e97c..dae7b08 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/Trigger.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/Trigger.java @@ -16,14 +16,65 @@ package com.easyagents.flow.core.chain.runtime; import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; public class Trigger implements Serializable { + private static final long serialVersionUID = 3165037658498721088L; + private String id; private String stateInstanceId; private String edgeId; private String nodeId; // 可以为 null,代表触发整个 chain private TriggerType type; private long triggerAt; // epoch ms + /** + * 当前运行时分配的触发器认领代际。 + * + *

字段名为兼容既有序列化数据保留。分布式仓储在触发器认领成功时分配, + * 并与该触发器租约共同续期和失效;该值不代表实例锁 fencing token。

+ */ + private long fencingToken; + /** + * 创建派生触发器时必须仍然有效的父触发器 fencing token。 + */ + private long requiredFencingToken; + /** + * 创建派生触发器时必须仍然有效的父实例锁 fencing token。 + */ + private long requiredLockFencingToken; + /** + * 创建派生触发器时必须仍然有效的父触发器 claim ID。 + */ + private String requiredFencingClaimId; + /** + * 基础设施投递失败次数,不占用业务节点重试次数。 + */ + private int deliveryAttempt; + /** + * 已完成业务终态收敛、等待可靠写入死信的标记。 + */ + private boolean deadLetterPending; + /** + * 待写入死信的稳定失败原因。 + */ + private String deadLetterReason; + /** + * 跨重试保持不变的逻辑执行 ID,用于副作用幂等键。 + */ + private String logicalExecutionId; + /** + * 可选执行通道,用于把会同步等待的子工作流与普通节点工作线程隔离。 + */ + private String executionLane; + /** + * 首个稳定入口意图携带的初始变量。 + * + *

仅用于实例仍处于 READY 时的崩溃恢复;正常启动提交后,运行时变量仍以 + * {@code ChainState.memory} 为唯一业务数据源。

+ */ + private Map startVariables; + private Map loopCursors; public Trigger() { } @@ -77,6 +128,260 @@ public class Trigger implements Serializable { this.triggerAt = triggerAt; } + /** + * 获取当前节点逻辑执行 ID。 + * + * @return 跨重试保持不变的逻辑执行 ID + */ + public String getLogicalExecutionId() { + return logicalExecutionId; + } + + /** + * 设置当前节点逻辑执行 ID。 + * + * @param logicalExecutionId 跨重试保持不变的逻辑执行 ID + */ + public void setLogicalExecutionId(String logicalExecutionId) { + this.logicalExecutionId = logicalExecutionId; + } + + /** + * 获取执行通道。 + * + * @return 通道名;{@code null} 表示默认通道 + */ + public String getExecutionLane() { + return executionLane; + } + + /** + * 设置执行通道。 + * + * @param executionLane 通道名 + */ + public void setExecutionLane(String executionLane) { + this.executionLane = executionLane; + } + + /** + * 获取崩溃恢复所需的初始变量。 + * + * @return 初始变量快照;未携带时为 {@code null} + */ + public Map getStartVariables() { + return startVariables; + } + + /** + * 设置崩溃恢复所需的初始变量。 + * + * @param startVariables 初始变量;仅首个稳定入口意图需要携带 + */ + public void setStartVariables(Map startVariables) { + this.startVariables = startVariables == null + ? null + : new LinkedHashMap<>(startVariables); + } + + /** + * 获取本次认领代际。 + * + * @return 单触发器认领代际;未认领时为 {@code 0} + */ + public long getFencingToken() { + return fencingToken; + } + + /** + * 设置本次认领代际。 + * + * @param fencingToken 单触发器认领代际 + */ + public void setFencingToken(long fencingToken) { + this.fencingToken = fencingToken; + } + + /** + * 获取保存派生触发器所依赖的父实例锁 fencing token。 + * + * @return 父实例锁 token;无锁约束时为 {@code 0} + */ + public long getRequiredLockFencingToken() { + return requiredLockFencingToken; + } + + /** + * 设置保存派生触发器所依赖的父实例锁 fencing token。 + * + * @param requiredLockFencingToken 父实例锁 token + */ + public void setRequiredLockFencingToken(long requiredLockFencingToken) { + this.requiredLockFencingToken = requiredLockFencingToken; + } + + /** + * 获取保存派生触发器所依赖的父 fencing token。 + * + * @return 父 fencing token;无父认领约束时为 {@code 0} + */ + public long getRequiredFencingToken() { + return requiredFencingToken; + } + + /** + * 设置保存派生触发器所依赖的父 fencing token。 + * + * @param requiredFencingToken 父 fencing token + */ + public void setRequiredFencingToken(long requiredFencingToken) { + this.requiredFencingToken = requiredFencingToken; + } + + /** + * 获取保存派生触发器所依赖的父 claim ID。 + * + * @return 父触发器 ID;无父认领约束时为 {@code null} + */ + public String getRequiredFencingClaimId() { + return requiredFencingClaimId; + } + + /** + * 设置保存派生触发器所依赖的父 claim ID。 + * + * @param requiredFencingClaimId 父触发器 ID + */ + public void setRequiredFencingClaimId(String requiredFencingClaimId) { + this.requiredFencingClaimId = requiredFencingClaimId; + } + + /** + * 获取基础设施投递失败次数。 + * + * @return 失败次数 + */ + public int getDeliveryAttempt() { + return deliveryAttempt; + } + + /** + * 设置基础设施投递失败次数。 + * + * @param deliveryAttempt 失败次数 + */ + public void setDeliveryAttempt(int deliveryAttempt) { + this.deliveryAttempt = deliveryAttempt; + } + + /** + * 判断触发器是否正在补写死信终态。 + * + * @return 等待死信持久化时为 {@code true} + */ + public boolean isDeadLetterPending() { + return deadLetterPending; + } + + /** + * 设置死信补写标记。 + * + * @param deadLetterPending 是否等待死信持久化 + */ + public void setDeadLetterPending(boolean deadLetterPending) { + this.deadLetterPending = deadLetterPending; + } + + /** + * 获取稳定死信原因。 + * + * @return 死信原因 + */ + public String getDeadLetterReason() { + return deadLetterReason; + } + + /** + * 设置稳定死信原因。 + * + * @param deadLetterReason 死信原因 + */ + public void setDeadLetterReason(String deadLetterReason) { + this.deadLetterReason = deadLetterReason; + } + + /** + * 获取触发器携带的循环代际游标。 + * + * @return 循环节点 ID 到游标的映射 + */ + public Map getLoopCursors() { + if (loopCursors == null) { + loopCursors = new LinkedHashMap<>(); + } + return loopCursors; + } + + /** + * 设置循环代际游标。 + * + * @param loopCursors 循环节点 ID 到游标的映射 + */ + public void setLoopCursors(Map loopCursors) { + this.loopCursors = loopCursors; + } + + /** + * 循环分支代际游标,用于拒绝过期或重复的父节点回调。 + */ + public static class LoopCursor implements Serializable { + private static final long serialVersionUID = 1L; + + private String resultId; + private int iterationIndex; + private String branchId; + + public LoopCursor() { + } + + /** + * 创建循环游标。 + * + * @param resultId 循环代际 ID + * @param iterationIndex 迭代序号 + * @param branchId 直属分支 ID + */ + public LoopCursor(String resultId, int iterationIndex, String branchId) { + this.resultId = resultId; + this.iterationIndex = iterationIndex; + this.branchId = branchId; + } + + public String getResultId() { + return resultId; + } + + public void setResultId(String resultId) { + this.resultId = resultId; + } + + public int getIterationIndex() { + return iterationIndex; + } + + public void setIterationIndex(int iterationIndex) { + this.iterationIndex = iterationIndex; + } + + public String getBranchId() { + return branchId; + } + + public void setBranchId(String branchId) { + this.branchId = branchId; + } + } + @Override public String toString() { return "Trigger{" + @@ -89,4 +394,3 @@ public class Trigger implements Serializable { '}'; } } - diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerClaimLostException.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerClaimLostException.java new file mode 100644 index 0000000..38c185c --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerClaimLostException.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + * Licensed under the GNU Lesser General Public License (LGPL), Version 3.0. + */ +package com.easyagents.flow.core.chain.runtime; + +/** + * 表示当前工作线程已经失去触发器租约,不再允许提交执行结果。 + */ +public class TriggerClaimLostException extends RuntimeException { + + /** + * 创建租约丢失异常。 + * + * @param triggerId 已失去租约的触发器 ID + */ + public TriggerClaimLostException(String triggerId) { + super("Workflow trigger claim ownership lost: " + triggerId); + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerScheduler.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerScheduler.java index 7d07a03..82bff43 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerScheduler.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerScheduler.java @@ -39,19 +39,49 @@ import java.util.concurrent.atomic.AtomicBoolean; public class TriggerScheduler { private static final Logger log = LoggerFactory.getLogger(TriggerScheduler.class); + private static final long CLAIM_LEASE_MS = TimeUnit.MINUTES.toMillis(1); + private static final long CLAIM_RENEW_INTERVAL_MS = TimeUnit.SECONDS.toMillis(20); + private static final int MAX_DELIVERY_ATTEMPTS = 20; + private static final long MAX_REDELIVERY_DELAY_MS = TimeUnit.MINUTES.toMillis(1); + /** + * 本地仅缓存一批定时任务;容量溢出时持久仓储仍是恢复与补偿来源。 + */ + private static final int MAX_LOCAL_SCHEDULED = 1024; + /** + * 精确定时只预热宽松近期限窗口,远期任务继续由持久仓储保管。 + */ + private static final long MIN_LOCAL_SCHEDULE_HORIZON_MS = + TimeUnit.MINUTES.toMillis(1); private final TriggerStore store; private final ScheduledExecutorService scheduler; private final ExecutorService worker; + private final Semaphore dispatchPermits; + private final ConcurrentMap laneWorkers = + new ConcurrentHashMap<>(); + private final ConcurrentMap laneDispatchPermits = + new ConcurrentHashMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); // map 用于管理取消:triggerId -> ScheduledFuture private final ConcurrentMap> scheduledFutures = new ConcurrentHashMap<>(); + /** + * 本地 Future 对应的绝对触发时间,用于容量满时保留更早到期任务。 + */ + private final ConcurrentMap scheduledTriggerTimes = + new ConcurrentHashMap<>(); + private final ConcurrentMap> claimRenewals = new ConcurrentHashMap<>(); + /** + * 串行化本地 Future 的容量检查与登记,确保并发调度时仍严格受容量上限约束。 + */ + private final Object localScheduleMonitor = new Object(); // consumer 来把 trigger 交给 ChainExecutor(或 ChainRuntime)去处理 private volatile TriggerConsumer consumer; + private volatile TriggerFailureListener failureListener; // 周期扫查间隔(ms) private final long scanIntervalMs; + private final long localScheduleHorizonMs; // 扫描任务 future private ScheduledFuture scanFuture; @@ -60,13 +90,34 @@ public class TriggerScheduler { void accept(Trigger trigger, ExecutorService worker); } + /** + * 触发器不可恢复失败监听器。 + */ + public interface TriggerFailureListener { + /** + * 在触发器 claim 仍有效时通知业务运行时收敛实例终态。 + * + * @param trigger 已死信触发器 + * @param failure 最后一次执行失败 + */ + boolean onDeadLetter(Trigger trigger, Throwable failure); + } + public TriggerScheduler(TriggerStore store, ScheduledExecutorService scheduler, ExecutorService worker, long scanIntervalMs) { this.store = Objects.requireNonNull(store, "TriggerStore required"); this.scheduler = Objects.requireNonNull(scheduler, "ScheduledExecutorService required"); this.worker = Objects.requireNonNull(worker, "ExecutorService required"); + this.dispatchPermits = createDispatchPermits(worker); this.scanIntervalMs = Math.max(1000, scanIntervalMs); + long scanHorizon = this.scanIntervalMs + > Long.MAX_VALUE / 3L + ? Long.MAX_VALUE + : this.scanIntervalMs * 3L; + this.localScheduleHorizonMs = Math.max( + MIN_LOCAL_SCHEDULE_HORIZON_MS, + scanHorizon); - // 恢复并 schedule + // 启动时只恢复已经到期的任务,避免把全部远期任务复制到本机 DelayQueue。 recoverAndSchedulePending(); // 启动周期扫查 findDue @@ -78,6 +129,40 @@ public class TriggerScheduler { this.consumer = consumer; } + /** + * 注册触发器不可恢复失败监听器。 + * + * @param listener 失败监听器 + */ + public void registerFailureListener(TriggerFailureListener listener) { + this.failureListener = listener; + } + + /** + * 注册独立执行通道。 + * + * @param lane 通道名 + * @param laneWorker 通道工作线程池 + */ + public void registerWorker( + String lane, ExecutorService laneWorker) { + if (lane == null || lane.isBlank()) { + throw new IllegalArgumentException("lane required"); + } + ExecutorService workerToRegister = + Objects.requireNonNull(laneWorker, "laneWorker required"); + ExecutorService previous = + laneWorkers.putIfAbsent(lane, workerToRegister); + if (previous != null && previous != workerToRegister) { + throw new IllegalStateException( + "Trigger worker lane already registered: " + lane); + } + Semaphore permits = createDispatchPermits(workerToRegister); + if (permits != null) { + laneDispatchPermits.putIfAbsent(lane, permits); + } + } + /** * schedule a trigger: persist -> schedule (单机语义) */ @@ -86,20 +171,55 @@ public class TriggerScheduler { if (trigger.getId() == null) { trigger.setId(UUID.randomUUID().toString()); } + if (trigger.getLogicalExecutionId() == null || trigger.getLogicalExecutionId().isBlank()) { + trigger.setLogicalExecutionId(trigger.getId()); + } store.save(trigger); scheduleInternal(trigger); return trigger; } + /** + * 仅在持久仓储中不存在同 ID 触发器时保存并调度。 + * + *

调用方需持有工作流实例锁;该方法用于可重放启动协议,稳定 ID 可避免 + * READY 到入口触发器持久化之间的崩溃窗口产生重复待执行任务。

+ * + * @param trigger 带稳定 ID 的触发器 + * @return 已存在或新保存的触发器 + */ + public Trigger scheduleIfAbsent(Trigger trigger) { + if (closed.get()) { + throw new IllegalStateException("TriggerScheduler closed"); + } + if (trigger == null || trigger.getId() == null + || trigger.getId().isBlank()) { + throw new IllegalArgumentException( + "Stable trigger ID required"); + } + if (trigger.getLogicalExecutionId() == null + || trigger.getLogicalExecutionId().isBlank()) { + trigger.setLogicalExecutionId( + trigger.getId()); + } + if (store.saveIfAbsent(trigger)) { + scheduleInternal(trigger); + return trigger; + } + Trigger existing = store.find( + trigger.getId()); + // 已有触发器可能刚好被其他 owner 认领;稳定入口仍视为已成功登记。 + return existing == null + ? trigger + : existing; + } + /** * cancel trigger (从 store 删除并尝试取消已 schedule 的 future) */ public boolean cancel(String triggerId) { boolean removed = store.remove(triggerId); - ScheduledFuture f = scheduledFutures.remove(triggerId); - if (f != null) { - f.cancel(false); - } + removeLocalSchedule(triggerId, true); return removed; } @@ -107,83 +227,185 @@ public class TriggerScheduler { * 主动触发(webhook/event/manual 场景) */ public boolean fire(String triggerId) { - if (closed.get()) return false; - Trigger t = store.find(triggerId); - if (t == null) return false; - if (consumer == null) { - // 无 consumer,仍从 store 中移除 - store.remove(triggerId); + if (closed.get()) { return false; } - // 在 worker 线程触发 consumer - worker.submit(() -> { - try { - consumer.accept(t, worker); - } catch (Exception e) { - log.error(e.toString(), e); - } finally { - // 默认语义:触发后移除 - store.remove(triggerId); - ScheduledFuture sf = scheduledFutures.remove(triggerId); - if (sf != null) sf.cancel(false); - } - }); - return true; + Trigger candidate = store.find(triggerId); + if (candidate == null) { + return false; + } + removeLocalSchedule(triggerId, true); + return claimAndDispatch(candidate); } /** * internal scheduling for a trigger (单机 scheduled semantics) */ private void scheduleInternal(Trigger trigger) { - if (closed.get()) return; - - long delay = Math.max(0, trigger.getTriggerAt() - System.currentTimeMillis()); - - // cancel any existing scheduled future for same id - ScheduledFuture prev = scheduledFutures.remove(trigger.getId()); - if (prev != null) prev.cancel(false); - - ScheduledFuture future = scheduler.schedule(() -> { - // double-check existence in store (可能已被 cancel) - Trigger existing = store.find(trigger.getId()); - if (existing == null) { - scheduledFutures.remove(trigger.getId()); + if (closed.get()) { + return; + } + long now = System.currentTimeMillis(); + if (trigger.getTriggerAt() + > scheduleHorizonTimestamp(now)) { + return; + } + synchronized (localScheduleMonitor) { + if (closed.get()) { return; } - - if (consumer != null) { - worker.submit(() -> { - try { - TriggerContext.setCurrentTrigger(existing); - consumer.accept(existing, worker); - } catch (Throwable e) { - log.error(e.toString(), e); - } finally { - TriggerContext.clearCurrentTrigger(); - store.remove(existing.getId()); - scheduledFutures.remove(existing.getId()); - } - }); - } else { - // 无 consumer,则移除 - store.remove(existing.getId()); - scheduledFutures.remove(existing.getId()); + ScheduledFuture existing; + while ((existing = scheduledFutures.get(trigger.getId())) != null) { + if (!existing.isDone() && !existing.isCancelled()) { + return; + } + // 已完成或取消的占位必须先原子移除,否则 putIfAbsent 会永久阻断重新调度。 + if (!scheduledFutures.remove(trigger.getId(), existing)) { + continue; + } + scheduledTriggerTimes.remove( + trigger.getId()); } - }, delay, TimeUnit.MILLISECONDS); + if (!hasLocalScheduleCapacity( + trigger.getTriggerAt())) { + return; + } + long delayMillis = Math.max( + 0L, + trigger.getTriggerAt() + - System.currentTimeMillis()); + ScheduledFuture future = scheduler.schedule( + () -> { + // 回调先在同一登记锁下同时移除两张索引,消除零延迟任务 + // 在 Future 与时间索引分步登记之间完成所造成的孤儿记录。 + removeLocalSchedule( + trigger.getId(), false); + claimAndDispatch(trigger); + }, + delayMillis, + TimeUnit.MILLISECONDS); + ScheduledFuture concurrent = + scheduledFutures.putIfAbsent( + trigger.getId(), future); + if (concurrent != null) { + future.cancel(false); + return; + } + scheduledTriggerTimes.put( + trigger.getId(), + trigger.getTriggerAt()); + // 零延迟任务可能在 put 前完成;完成态二次清理避免残留无效 Future。 + if (future.isDone() || future.isCancelled()) { + scheduledFutures.remove( + trigger.getId(), future); + scheduledTriggerTimes.remove( + trigger.getId()); + } + } + } - scheduledFutures.put(trigger.getId(), future); + /** + * 检查本地调度容量,并在容量耗尽时清理已完成或已取消的占位。 + * + *

正常路径只执行常量时间判断;达到上限时最多扫描 + * {@link #MAX_LOCAL_SCHEDULED} 个条目,避免极窄竞态残留导致永久停摆。

+ * + * @param triggerAt 待登记任务的绝对触发时间 + * @return 仍可接收本地到期任务时为 {@code true} + */ + private boolean hasLocalScheduleCapacity( + long triggerAt) { + if (scheduledFutures.size() < MAX_LOCAL_SCHEDULED) { + return true; + } + for (Map.Entry> entry : scheduledFutures.entrySet()) { + ScheduledFuture future = entry.getValue(); + if (future.isDone() || future.isCancelled()) { + if (scheduledFutures.remove( + entry.getKey(), future)) { + scheduledTriggerTimes.remove( + entry.getKey()); + } + } + } + if (scheduledFutures.size() + < MAX_LOCAL_SCHEDULED) { + return true; + } + Map.Entry latest = null; + for (Map.Entry entry + : scheduledTriggerTimes.entrySet()) { + if (latest == null + || entry.getValue() + > latest.getValue()) { + latest = entry; + } + } + if (latest == null + || latest.getValue() <= triggerAt) { + return false; + } + removeLocalSchedule( + latest.getKey(), true); + return scheduledFutures.size() + < MAX_LOCAL_SCHEDULED; + } + + /** + * 计算本轮应预热到本机的最远触发时间。 + * + * @return 当前时间加本地近期限窗口;溢出时为最大时间戳 + */ + private long scheduleHorizonTimestamp() { + return scheduleHorizonTimestamp( + System.currentTimeMillis()); + } + + /** + * 基于给定时间计算本机预热边界。 + * + * @param now 当前时间戳 + * @return 当前时间加近期限窗口;溢出时为最大时间戳 + */ + private long scheduleHorizonTimestamp( + long now) { + return now > Long.MAX_VALUE + - localScheduleHorizonMs + ? Long.MAX_VALUE + : now + localScheduleHorizonMs; + } + + /** + * 同时移除本地 Future 与其触发时间索引。 + * + * @param triggerId 触发器 ID + * @param cancel 是否取消尚未完成的 Future + * @return 被移除的 Future;不存在时为 {@code null} + */ + private ScheduledFuture removeLocalSchedule( + String triggerId, + boolean cancel) { + synchronized (localScheduleMonitor) { + ScheduledFuture scheduled = + scheduledFutures.remove(triggerId); + scheduledTriggerTimes.remove(triggerId); + if (cancel && scheduled != null) { + scheduled.cancel(false); + } + return scheduled; + } } private void recoverAndSchedulePending() { try { - List list = store.findAllPending(); + List list = store.findDue( + scheduleHorizonTimestamp()); if (list == null || list.isEmpty()) return; for (Trigger t : list) { scheduleInternal(t); } } catch (Throwable t) { - // 忽略单次恢复错误,继续运行 - t.printStackTrace(); + log.error("Failed to recover pending workflow triggers", t); } } @@ -191,7 +413,7 @@ public class TriggerScheduler { if (closed.get()) return; scanFuture = scheduler.scheduleAtFixedRate(() -> { try { - long upto = System.currentTimeMillis(); + long upto = scheduleHorizonTimestamp(); List due = store.findDue(upto); if (due == null || due.isEmpty()) return; for (Trigger t : due) { @@ -200,27 +422,363 @@ public class TriggerScheduler { if (sf != null && !sf.isDone() && !sf.isCancelled()) { continue; } - // 直接提交到 worker,让 consumer 处理;并从 store 中移除 - if (consumer != null) { - worker.submit(() -> { - try { - consumer.accept(t, worker); - } finally { - store.remove(t.getId()); - ScheduledFuture f2 = scheduledFutures.remove(t.getId()); - if (f2 != null) f2.cancel(false); - } - }); - } else { - store.remove(t.getId()); - } + scheduleInternal(t); } } catch (Throwable tt) { - tt.printStackTrace(); + log.error("Failed to scan due workflow triggers", tt); } }, scanIntervalMs, scanIntervalMs, TimeUnit.MILLISECONDS); } + /** + * 原子认领触发器并提交给工作线程。 + * + * @param candidate 已加载的候选触发器 + * @return 成功认领并提交时为 true + */ + private boolean claimAndDispatch(Trigger candidate) { + if (candidate == null || candidate.getId() == null) { + return false; + } + String triggerId = candidate.getId(); + ExecutorService dispatchWorker = resolveWorker(candidate); + Semaphore selectedPermits = resolveDispatchPermits(candidate); + if (selectedPermits != null && !selectedPermits.tryAcquire()) { + // 保留持久化触发器并清理本地占位,让后续扫描能够重新调度。 + removeLocalSchedule(triggerId, false); + return false; + } + Trigger claimed; + try { + claimed = store.claim(candidate, CLAIM_LEASE_MS); + } catch (RuntimeException | Error error) { + removeLocalSchedule(triggerId, false); + releaseDispatchPermit(selectedPermits); + throw error; + } + if (claimed == null) { + releaseDispatchPermit(selectedPermits); + removeLocalSchedule(triggerId, false); + return false; + } + TriggerConsumer currentConsumer = consumer; + if (currentConsumer == null) { + releaseClaimBestEffort(claimed, "consumer is unavailable"); + releaseDispatchPermit(selectedPermits); + removeLocalSchedule(triggerId, false); + return false; + } + + ScheduledFuture renewal; + try { + renewal = scheduler.scheduleAtFixedRate( + () -> renewClaim(claimed), + CLAIM_RENEW_INTERVAL_MS, + CLAIM_RENEW_INTERVAL_MS, + TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException error) { + releaseClaimBestEffort(claimed, "claim renewal scheduling was rejected"); + releaseDispatchPermit(selectedPermits); + removeLocalSchedule(triggerId, false); + return false; + } + ScheduledFuture existingRenewal = claimRenewals.put(claimed, renewal); + if (existingRenewal != null) { + existingRenewal.cancel(false); + } + + try { + dispatchWorker.submit(() -> consumeClaimedTrigger( + claimed, + currentConsumer, + dispatchWorker, + selectedPermits)); + return true; + } catch (RejectedExecutionException error) { + cancelClaimRenewal(claimed); + releaseClaimBestEffort(claimed, "worker submission was rejected"); + releaseDispatchPermit(selectedPermits); + removeLocalSchedule(triggerId, false); + log.error("Workflow trigger worker rejected task, triggerId={}", triggerId, error); + return false; + } + } + + /** + * 执行已认领触发器,并按执行结果确认或释放。 + * + * @param trigger 已认领触发器 + * @param currentConsumer 本次执行使用的消费者快照 + */ + private void consumeClaimedTrigger( + Trigger trigger, + TriggerConsumer currentConsumer, + ExecutorService dispatchWorker, + Semaphore selectedPermits) { + boolean succeeded = false; + boolean deadLetter = false; + boolean retryWithoutDeadLetter = false; + boolean rescheduleReleasedTrigger = false; + Throwable failure = null; + try { + TriggerContext.setCurrentTrigger(trigger); + if (trigger.isDeadLetterPending()) { + deadLetter = true; + failure = new NonRetryableTriggerException( + trigger.getDeadLetterReason()); + } else { + currentConsumer.accept(trigger, dispatchWorker); + succeeded = true; + } + } catch (NonRetryableTriggerException error) { + failure = error; + deadLetter = true; + log.error("Workflow trigger is not retryable, triggerId={}", trigger.getId(), error); + } catch (RetryableTriggerException error) { + failure = error; + retryWithoutDeadLetter = true; + log.warn( + "Workflow trigger hit transient infrastructure contention, triggerId={}", + trigger.getId()); + } catch (Throwable error) { + failure = error; + log.error("Workflow trigger execution failed, triggerId={}", trigger.getId(), error); + } finally { + try { + if (succeeded) { + store.acknowledge(trigger); + } else { + int deliveryAttempt = trigger.isDeadLetterPending() + ? Math.max(1, trigger.getDeliveryAttempt()) + : incrementDeliveryAttempt(trigger); + if (retryWithoutDeadLetter) { + trigger.setTriggerAt( + System.currentTimeMillis() + + redeliveryDelayMillis( + deliveryAttempt)); + store.release(trigger); + rescheduleReleasedTrigger = true; + } else if (deadLetter + || deliveryAttempt + >= MAX_DELIVERY_ATTEMPTS) { + String reason = failure == null + ? "delivery attempts exhausted" + : failure.getClass().getName() + + ": " + + failure.getMessage(); + if (trigger.isDeadLetterPending() + && trigger.getDeadLetterReason() != null) { + reason = trigger.getDeadLetterReason(); + } else { + trigger.setDeadLetterPending(true); + trigger.setDeadLetterReason(reason); + store.markDeadLetterPending( + trigger); + } + if (!store.renewClaim( + trigger, CLAIM_LEASE_MS)) { + throw new TriggerClaimLostException( + trigger.getId()); + } + TriggerFailureListener currentFailureListener = + failureListener; + if (currentFailureListener != null + && !currentFailureListener.onDeadLetter( + trigger, failure)) { + trigger.setTriggerAt( + System.currentTimeMillis() + + MAX_REDELIVERY_DELAY_MS); + store.release(trigger); + rescheduleReleasedTrigger = true; + return; + } + store.deadLetter(trigger, reason); + } else { + trigger.setTriggerAt( + System.currentTimeMillis() + + redeliveryDelayMillis( + deliveryAttempt)); + store.release(trigger); + rescheduleReleasedTrigger = true; + } + } + } catch (Throwable terminalStoreError) { + if (trigger.isDeadLetterPending()) { + releaseDeadLetterPendingBestEffort(trigger); + } + log.error( + "Failed to finalize workflow trigger, triggerId={}", + trigger.getId(), + terminalStoreError); + } finally { + TriggerContext.clearCurrentTrigger(); + cancelClaimRenewal(trigger); + removeLocalSchedule( + trigger.getId(), false); + releaseDispatchPermit(selectedPermits); + if (rescheduleReleasedTrigger) { + scheduleInternal(trigger); + } + } + } + } + + /** + * 在仍持有 claim 时保存待补写死信标记,避免业务终态成功后被普通 ACK 吞掉。 + * + * @param trigger 待补写死信的触发器 + */ + private void releaseDeadLetterPendingBestEffort( + Trigger trigger) { + try { + if (!store.renewClaim(trigger, CLAIM_LEASE_MS)) { + return; + } + trigger.setTriggerAt( + System.currentTimeMillis() + + MAX_REDELIVERY_DELAY_MS); + store.release(trigger); + } catch (Throwable releaseError) { + log.error( + "Failed to persist pending dead-letter finalization, triggerId={}", + trigger.getId(), + releaseError); + } + } + + /** + * 尽力释放已认领触发器。 + * + *

释放失败时保留 Redis 中的触发器和租约,等待租约自然过期恢复。调用方仍可继续 + * 清理本地调度状态和归还容量许可。

+ * + * @param trigger 已认领触发器 + * @param reason 释放原因 + */ + private void releaseClaimBestEffort(Trigger trigger, String reason) { + try { + store.release(trigger); + } catch (Throwable error) { + log.error( + "Failed to release workflow trigger, triggerId={}, reason={}", + trigger == null ? null : trigger.getId(), + reason, + error); + } + } + + /** + * 增加基础设施投递失败次数。 + * + * @param trigger 当前触发器 + * @return 增加后的次数 + */ + private int incrementDeliveryAttempt(Trigger trigger) { + int attempt = Math.max(0, trigger.getDeliveryAttempt()) + 1; + trigger.setDeliveryAttempt(attempt); + return attempt; + } + + /** + * 计算带上限的指数退避,避免缺失定义或短暂冲突形成热循环。 + * + * @param attempt 已失败次数 + * @return 下次投递延迟毫秒数 + */ + private long redeliveryDelayMillis(int attempt) { + int shift = Math.min(16, Math.max(0, attempt - 1)); + long delay = 1_000L << shift; + return Math.min(MAX_REDELIVERY_DELAY_MS, delay); + } + + /** + * 根据工作线程池的真实容量建立领取前背压。 + * + * @param executor 工作线程池 + * @return 容量信号量;无法识别容量时返回 null + */ + private Semaphore createDispatchPermits(ExecutorService executor) { + if (!(executor instanceof ThreadPoolExecutor)) { + return null; + } + ThreadPoolExecutor pool = (ThreadPoolExecutor) executor; + long capacity = (long) pool.getMaximumPoolSize() + pool.getQueue().remainingCapacity(); + return new Semaphore((int) Math.max(1L, Math.min(Integer.MAX_VALUE, capacity))); + } + + /** + * 释放一个工作线程容量许可。 + */ + private void releaseDispatchPermit(Semaphore permits) { + if (permits != null) { + permits.release(); + } + } + + /** + * 解析触发器执行线程池。 + * + * @param trigger 触发器 + * @return 默认或独立通道线程池 + */ + private ExecutorService resolveWorker(Trigger trigger) { + String lane = trigger == null ? null : trigger.getExecutionLane(); + return lane == null ? worker : laneWorkers.getOrDefault(lane, worker); + } + + /** + * 解析所选线程池对应的领取前背压许可。 + * + * @param trigger 触发器 + * @return 容量许可;无法识别时为 {@code null} + */ + private Semaphore resolveDispatchPermits(Trigger trigger) { + String lane = trigger == null ? null : trigger.getExecutionLane(); + return lane == null + ? dispatchPermits + : laneDispatchPermits.getOrDefault(lane, dispatchPermits); + } + + /** + * 续期触发器认领租约。 + * + * @param trigger 已认领触发器 + */ + private void renewClaim(Trigger trigger) { + try { + if (!store.renewClaim(trigger, CLAIM_LEASE_MS)) { + log.warn("Workflow trigger claim renewal lost ownership, triggerId={}", trigger.getId()); + cancelClaimRenewal(trigger); + } + } catch (Throwable error) { + log.error("Workflow trigger claim renewal failed, triggerId={}", trigger.getId(), error); + } + } + + /** + * 在业务状态提交前验证具体触发器仍由当前工作线程持有。 + * + * @param trigger 已认领触发器 + * @throws TriggerClaimLostException 租约已失效或已转移 + */ + public void assertClaimOwned(Trigger trigger) { + if (trigger != null && !store.renewClaim(trigger, CLAIM_LEASE_MS)) { + throw new TriggerClaimLostException(trigger.getId()); + } + } + + /** + * 取消触发器租约续期任务。 + * + * @param trigger 已认领触发器 + */ + private void cancelClaimRenewal(Trigger trigger) { + ScheduledFuture renewal = claimRenewals.remove(trigger); + if (renewal != null) { + renewal.cancel(false); + } + } + public void shutdown() { if (closed.compareAndSet(false, true)) { if (scanFuture != null) scanFuture.cancel(false); @@ -232,6 +790,11 @@ public class TriggerScheduler { } } scheduledFutures.clear(); + scheduledTriggerTimes.clear(); + for (ScheduledFuture renewal : claimRenewals.values()) { + renewal.cancel(false); + } + claimRenewals.clear(); try { scheduler.shutdownNow(); @@ -241,6 +804,18 @@ public class TriggerScheduler { worker.shutdownNow(); } catch (Throwable ignored) { } + for (ExecutorService laneWorker : + new java.util.HashSet<>(laneWorkers.values())) { + if (laneWorker == worker) { + continue; + } + try { + laneWorker.shutdownNow(); + } catch (Throwable ignored) { + } + } + laneWorkers.clear(); + laneDispatchPermits.clear(); } } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerStore.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerStore.java index fdb3dd9..02a2562 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerStore.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/TriggerStore.java @@ -18,8 +18,27 @@ package com.easyagents.flow.core.chain.runtime; import java.util.List; public interface TriggerStore { + Trigger save(Trigger trigger); + /** + * 仅在同 ID 触发器尚不存在时原子保存。 + * + *

缺省实现保证同一仓储实例内原子;分布式仓储必须覆盖为跨进程原子操作。

+ * + * @param trigger 带稳定 ID 的触发器 + * @return 本次成功创建时为 {@code true},已存在时为 {@code false} + */ + default boolean saveIfAbsent(Trigger trigger) { + synchronized (this) { + if (find(trigger.getId()) != null) { + return false; + } + save(trigger); + return true; + } + } + boolean remove(String triggerId); Trigger find(String triggerId); @@ -27,4 +46,116 @@ public interface TriggerStore { List findDue(long uptoTimestamp); List findAllPending(); + + /** + * 原子认领待执行触发器。 + *

+ * 缺省实现适用于单进程仓储:先读取再以删除结果作为认领成功标志。 + * + * @param triggerId 触发器 ID + * @param leaseMillis 认领租约毫秒数 + * @return 认领成功时返回触发器,否则返回 null + */ + default Trigger claim(String triggerId, long leaseMillis) { + Trigger trigger = find(triggerId); + return trigger != null && remove(triggerId) ? trigger : null; + } + + /** + * 原子认领已加载的待执行触发器。 + * + *

分布式仓储可使用候选触发器中的实例 ID 构建与本次 claim 绑定的执行守卫, + * 避免认领前额外读取完整触发器负载。

+ * + * @param candidate 扫描或主动触发阶段已加载的候选触发器 + * @param leaseMillis 认领租约毫秒数 + * @return 认领成功时返回触发器,否则返回 {@code null} + */ + default Trigger claim(Trigger candidate, long leaseMillis) { + return candidate == null ? null : claim(candidate.getId(), leaseMillis); + } + + /** + * 仅按 ID 续期触发器租约。 + * + *

分布式仓储无法仅凭 ID 验证 owner token。该兼容入口不应再用于运行时提交路径, + * 调用方必须保留 {@link Trigger} 认领对象并使用 + * {@link #renewClaim(Trigger, long)}。

+ * + * @param triggerId 触发器 ID + * @param leaseMillis 新租约毫秒数 + * @return 不适用 + * @throws UnsupportedOperationException 始终抛出,防止无 owner token 的不安全续期 + */ + @Deprecated + default boolean renewClaim(String triggerId, long leaseMillis) { + throw new UnsupportedOperationException("Trigger claim token is required"); + } + + /** + * 续期调用方持有的具体触发器租约。 + * + * @param trigger 已认领触发器对象 + * @param leaseMillis 新租约毫秒数 + * @return 续期成功时为 true + */ + default boolean renewClaim(Trigger trigger, long leaseMillis) { + // 单进程缺省仓储在 claim 时已经移除触发器,不需要租约续期。 + return true; + } + + /** + * 仅按 ID 确认触发器。 + * + *

分布式仓储无法仅凭 ID 验证 owner token,运行时必须使用 + * {@link #acknowledge(Trigger)}。

+ * + * @param triggerId 触发器 ID + * @throws UnsupportedOperationException 始终抛出,防止旧 owner 删除新 owner 的任务 + */ + @Deprecated + default void acknowledge(String triggerId) { + throw new UnsupportedOperationException("Claimed trigger object is required"); + } + + /** + * 确认调用方持有的具体触发器执行成功。 + * + * @param trigger 已认领触发器对象 + */ + default void acknowledge(Trigger trigger) { + // 单进程缺省认领已经移除触发器,无需再次处理。 + } + + /** + * 释放失败执行的触发器,使其可以再次被认领。 + * + * @param trigger 执行失败的触发器 + */ + default void release(Trigger trigger) { + save(trigger); + } + + /** + * 在保持当前 claim 的同时持久化待补写死信标记。 + * + *

分布式仓储必须校验具体 owner token;进程崩溃后,新 owner 依靠该标记 + * 跳过业务执行并继续终态协议。

+ * + * @param trigger 已认领且标记为待补写死信的触发器 + */ + default void markDeadLetterPending( + Trigger trigger) { + // 单进程仓储的 claimed trigger 仅存在于当前调用栈,无需额外持久化。 + } + + /** + * 将不可继续执行或超过投递上限的触发器移入死信。 + * + * @param trigger 已认领触发器 + * @param reason 死信原因 + */ + default void deadLetter(Trigger trigger, String reason) { + acknowledge(trigger); + } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java index 9d94608..040087e 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java @@ -16,6 +16,8 @@ package com.easyagents.flow.core.code.impl; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.NodeState; import com.easyagents.flow.core.code.CodeRuntimeEngine; import com.easyagents.flow.core.node.CodeNode; import com.easyagents.flow.core.util.graalvm.JsInteropUtils; @@ -39,8 +41,13 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { public Map execute(String code, CodeNode node, Chain chain) { try (Context context = CONTEXT_BUILDER.build()) { Value bindings = context.getBindings("js"); + ChainState chainState = + chain.getExecutionState(); + NodeState nodeState = + chain.getNodeState(node.getId()); - Map all = chain.getState().getMemory(); + Map all = + chainState.getMemory(); all.forEach((key, value) -> { if (!key.contains(".")) { bindings.putMember(key, JsInteropUtils.wrapJavaValueForJS(context, value)); @@ -48,23 +55,20 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { }); // 注入参数 - Map parameterValues = chain.getState().resolveParameters(node); + Map parameterValues = + chainState.resolveParameters(node); if (parameterValues != null) { for (Map.Entry entry : parameterValues.entrySet()) { bindings.putMember(entry.getKey(), JsInteropUtils.wrapJavaValueForJS(context, entry.getValue())); } } - bindings.putMember("_chain", chain); - bindings.putMember("_state", chain.getNodeState(node.getId())); - - // 在 JS 中创建 _result 对象 context.eval("js", "var _result = {};"); // 注入 _chain 和 _context bindings.putMember("_chain", chain); - bindings.putMember("_state", chain.getNodeState(node.getId())); + bindings.putMember("_state", nodeState); // 执行用户脚本 context.eval("js", code); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java index 551cc33..d681115 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java @@ -85,6 +85,7 @@ public interface Llm { * 实现了Serializable接口,支持序列化 */ class ChatOptions implements Serializable { + private static final long serialVersionUID = 1L; private String seed; private Float temperature = 0.8f; diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/BaseNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/BaseNode.java index c8ca930..205dd81 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/BaseNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/BaseNode.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; public abstract class BaseNode extends Node { + private static final long serialVersionUID = 1L; protected List parameters; protected List outputDefs; diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java index 65f5201..09e1f3f 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; public class CodeNode extends BaseNode { + private static final long serialVersionUID = 1L; + protected String engine; protected String code; @@ -52,7 +54,8 @@ public class CodeNode extends BaseNode { throw new IllegalArgumentException("code is empty"); } - ChainState chainState = chain.getState(); + ChainState chainState = + chain.getExecutionState(); Map parameterValues = chainState.resolveParameters(this); String newCode = TextTemplate.of(code).formatToString(chainState.buildTemplateRootMaps(parameterValues)); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/ConfirmNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/ConfirmNode.java index 52682c1..c1f17ba 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/ConfirmNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/ConfirmNode.java @@ -25,6 +25,8 @@ import com.easyagents.flow.core.chain.repository.ChainStateField; import java.util.*; public class ConfirmNode extends BaseNode { + private static final long serialVersionUID = 1L; + private String message; private List confirms; @@ -70,7 +72,8 @@ public class ConfirmNode extends BaseNode { Map values; try { - values = chain.getState().resolveParameters(this, confirmParameters); + values = chain.getExecutionState() + .resolveParameters(this, confirmParameters); // 移除 confirm 参数,方便在其他节点二次确认,或者在 for 循环中第二次获取 chain.updateStateSafely(state -> { for (Parameter confirmParameter : confirmParameters) { @@ -94,7 +97,12 @@ public class ConfirmNode extends BaseNode { } // 获取参数值,不会触发 ChainSuspendException 错误 - Map parameterValues = chain.getState().resolveParameters(this, newParameters, null, true); + Map parameterValues = + chain.getExecutionState().resolveParameters( + this, + newParameters, + null, + true); // 设置 enums,方便前端给用户进行选择 for (Parameter confirmParameter : confirmParameters) { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/EndNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/EndNode.java index c6e39f2..4a645ce 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/EndNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/EndNode.java @@ -22,6 +22,8 @@ import java.util.HashMap; import java.util.Map; public class EndNode extends BaseNode { + private static final long serialVersionUID = 1L; + private boolean normal = true; private String message; @@ -47,7 +49,8 @@ public class EndNode extends BaseNode { @Override public Map execute(Chain chain) { - + ChainState chainState = + chain.getExecutionState(); Map output = new HashMap<>(); if (normal) { output.put(ChainConsts.CHAIN_STATE_STATUS_KEY, ChainStatus.SUCCEEDED); @@ -62,7 +65,7 @@ public class EndNode extends BaseNode { if (this.outputDefs != null) { for (Parameter outputDef : this.outputDefs) { if (outputDef.getRefType() == RefType.REF) { - output.put(outputDef.getName(), chain.getState().resolveValue(outputDef.getRef())); + output.put(outputDef.getName(), chainState.resolveValue(outputDef.getRef())); } else if (outputDef.getRefType() == RefType.INPUT) { output.put(outputDef.getName(), outputDef.getRef()); } else if (outputDef.getRefType() == RefType.FIXED) { @@ -70,7 +73,7 @@ public class EndNode extends BaseNode { } // default is ref type else if (StringUtil.hasText(outputDef.getRef())) { - output.put(outputDef.getName(), chain.getState().resolveValue(outputDef.getRef())); + output.put(outputDef.getName(), chainState.resolveValue(outputDef.getRef())); } } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/HttpNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/HttpNode.java index dfbc5d3..f051de7 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/HttpNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/HttpNode.java @@ -22,21 +22,33 @@ import com.easyagents.flow.core.chain.DataType; import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.filestoreage.FileStorage; import com.easyagents.flow.core.filestoreage.FileStorageManager; +import com.easyagents.flow.core.util.IoBulkhead; import com.easyagents.flow.core.util.OkHttpClientUtil; import com.easyagents.flow.core.util.StringUtil; import com.easyagents.flow.core.util.TextTemplate; import okhttp3.*; +import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; public class HttpNode extends BaseNode { + private static final long serialVersionUID = 1L; + + + private static final long DEFAULT_MAX_TEXT_RESPONSE_BYTES = 64L * 1024L * 1024L; + private static final long DEFAULT_MAX_FILE_RESPONSE_BYTES = 2L * 1024L * 1024L * 1024L; + private static final String MAX_RESPONSE_BYTES_PROPERTY = "tinyflow.http.max-response-bytes"; + private static final String MAX_FILE_RESPONSE_BYTES_PROPERTY = "tinyflow.http.max-file-response-bytes"; private String url; private String method; @@ -145,7 +157,7 @@ public class HttpNode extends BaseNode { @Override public Map execute(Chain chain) { - int maxRetry = 5; + int maxRetry = supportsAutomaticRetry(method) ? 5 : 1; long retryInterval = 2000L; int attempt = 0; @@ -161,7 +173,7 @@ public class HttpNode extends BaseNode { lastError = ex; // 判断是否需要重试 - if (!shouldRetry(ex)) { + if (attempt >= maxRetry || !shouldRetry(ex)) { throw wrapAsRuntime(ex, attempt); } @@ -199,6 +211,19 @@ public class HttpNode extends BaseNode { return cause instanceof IOException; } + /** + * 判断当前方法是否允许节点内部自动重试。 + * + * @param requestMethod HTTP 方法 + * @return 仅无业务副作用的读取类方法返回 {@code true} + */ + protected boolean supportsAutomaticRetry(String requestMethod) { + return StringUtil.noText(requestMethod) + || "GET".equalsIgnoreCase(requestMethod) + || "HEAD".equalsIgnoreCase(requestMethod) + || "OPTIONS".equalsIgnoreCase(requestMethod); + } + private RuntimeException wrapAsRuntime(Throwable ex, int attempt) { if (ex instanceof RuntimeException) { return (RuntimeException) ex; @@ -212,13 +237,18 @@ public class HttpNode extends BaseNode { public Map doExecute(Chain chain) throws IOException { - Map argsMap = chain.getState().resolveParameters(this); + Map argsMap = + chain.getExecutionState().resolveParameters(this); String newUrl = TextTemplate.of(url) - .formatToString(chain.getState().buildTemplateRootMaps(argsMap)); + .formatToString( + chain.getExecutionState() + .buildTemplateRootMaps(argsMap)); Request.Builder reqBuilder = new Request.Builder().url(newUrl); - Map headersMap = chain.getState().resolveParameters(this, headers, argsMap); + Map headersMap = + chain.getExecutionState().resolveParameters( + this, headers, argsMap); headersMap.forEach((s, o) -> reqBuilder.addHeader(s, String.valueOf(o))); if (StringUtil.noText(method) || "GET".equalsIgnoreCase(method)) { @@ -227,8 +257,12 @@ public class HttpNode extends BaseNode { reqBuilder.method(method.toUpperCase(), getRequestBody(chain, argsMap)); } - OkHttpClient okHttpClient = OkHttpClientUtil.buildDefaultClient(); - try (Response response = okHttpClient.newCall(reqBuilder.build()).execute()) { + // 节点层统一计算总尝试次数,禁用客户端隐式重试,避免乘法式放大。 + OkHttpClient okHttpClient = + OkHttpClientUtil.buildNoRetryClient(); + // 共享客户端拦截器持有请求许可直到响应体关闭,避免同一请求重复领取许可。 + try (Response response = okHttpClient.newCall( + reqBuilder.build()).execute()) { // 服务器异常 if (response.code() >= 500 && response.code() < 600) { @@ -263,17 +297,39 @@ public class HttpNode extends BaseNode { } if (bodyDataType == null) { - result.put("body", body.string()); + try (IoBulkhead.Permit ignored = + IoBulkhead.responseAggregation().acquire( + IoBulkhead.targetForUrl(newUrl))) { + result.put("body", readTextBody( + body, resolveMaxTextResponseBytes())); + } } else if (bodyDataType == DataType.Object || bodyDataType.getValue().startsWith("Array")) { - result.put("body", JSON.parse(body.string())); + try (IoBulkhead.Permit ignored = + IoBulkhead.responseAggregation().acquire( + IoBulkhead.targetForUrl(newUrl))) { + result.put("body", JSON.parse(readTextBody( + body, resolveMaxTextResponseBytes()))); + } } else if (bodyDataType == DataType.File) { - try (InputStream stream = body.byteStream()) { + long maxFileResponseBytes = resolveMaxFileResponseBytes(); + validateDeclaredResponseSize(body, maxFileResponseBytes); + try (InputStream stream = limitResponseStream(body.byteStream(), maxFileResponseBytes)) { FileStorage fileStorage = FileStorageManager.getInstance().getFileStorage(); - String fileUrl = fileStorage.saveFile(stream, responseHeaders, this, chain); - result.put("body", fileUrl); + try (IoBulkhead.Permit ignored = + IoBulkhead.storage().acquire( + "storage:http-response")) { + String fileUrl = fileStorage.saveFile( + stream, responseHeaders, this, chain); + result.put("body", fileUrl); + } } } else { - result.put("body", body.string()); + try (IoBulkhead.Permit ignored = + IoBulkhead.responseAggregation().acquire( + IoBulkhead.targetForUrl(newUrl))) { + result.put("body", readTextBody( + body, resolveMaxTextResponseBytes())); + } } return result; } @@ -282,19 +338,26 @@ public class HttpNode extends BaseNode { private RequestBody getRequestBody(Chain chain, Map formatArgs) { if ("json".equals(bodyType)) { String bodyJsonString = TextTemplate.of(bodyJson) - .formatToString(chain.getState().buildTemplateContextMap(formatArgs), true); + .formatToString( + chain.getExecutionState() + .buildTemplateRootMaps(formatArgs), + true); JSONObject jsonObject = JSON.parseObject(bodyJsonString); return RequestBody.create(jsonObject.toString(), MediaType.parse("application/json")); } if ("x-www-form-urlencoded".equals(bodyType)) { - Map formUrlencodedMap = chain.getState().resolveParameters(this, formUrlencoded); + Map formUrlencodedMap = + chain.getExecutionState().resolveParameters( + this, formUrlencoded); String bodyString = mapToQueryString(formUrlencodedMap); return RequestBody.create(bodyString, MediaType.parse("application/x-www-form-urlencoded")); } if ("form-data".equals(bodyType)) { - Map formDataMap = chain.getState().resolveParameters(this, formData, formatArgs); + Map formDataMap = + chain.getExecutionState().resolveParameters( + this, formData, formatArgs); MultipartBody.Builder builder = new MultipartBody.Builder() .setType(MultipartBody.FORM); @@ -320,13 +383,97 @@ public class HttpNode extends BaseNode { if ("raw".equals(bodyType)) { String rawBodyString = TextTemplate.of(rawBody) - .formatToString(chain.getState().buildTemplateRootMaps(formatArgs)); + .formatToString( + chain.getExecutionState() + .buildTemplateRootMaps(formatArgs)); return RequestBody.create(rawBodyString, null); } //none return RequestBody.create("", null); } + /** + * 在宽松响应大小保护下读取文本响应。 + * + * @param body HTTP 响应体 + * @return 响应文本 + * @throws IOException 响应读取失败或超出限制时抛出 + */ + protected String readTextBody(ResponseBody body, long maxResponseBytes) throws IOException { + validateDeclaredResponseSize(body, maxResponseBytes); + Charset charset = body.contentType() == null + ? StandardCharsets.UTF_8 + : body.contentType().charset(StandardCharsets.UTF_8); + int initialCapacity = body.contentLength() > 0L + ? (int) Math.min(body.contentLength(), 64L * 1024L) + : 8 * 1024; + try (InputStream input = limitResponseStream(body.byteStream(), maxResponseBytes); + ByteArrayOutputStream output = new ByteArrayOutputStream(initialCapacity)) { + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return output.toString(charset); + } + } + + /** + * 校验响应声明长度,避免继续处理已知超限内容。 + * + * @param body HTTP 响应体 + * @param maxResponseBytes 最大允许字节数 + * @throws IOException 声明长度超过限制时抛出 + */ + private void validateDeclaredResponseSize(ResponseBody body, long maxResponseBytes) throws IOException { + long contentLength = body.contentLength(); + if (maxResponseBytes > 0L && contentLength > maxResponseBytes) { + throw responseSizeExceeded(maxResponseBytes); + } + } + + /** + * 为响应流增加按实际读取字节数执行的限制。 + * + * @param inputStream 原始响应流 + * @param maxResponseBytes 最大允许字节数 + * @return 受限响应流 + */ + private InputStream limitResponseStream(InputStream inputStream, long maxResponseBytes) { + if (maxResponseBytes <= 0L) { + return inputStream; + } + return new LimitedResponseInputStream(inputStream, maxResponseBytes); + } + + /** + * 获取文本或 JSON 响应字节数上限。 + * + * @return 最大允许字节数 + */ + private long resolveMaxTextResponseBytes() { + return Long.getLong(MAX_RESPONSE_BYTES_PROPERTY, DEFAULT_MAX_TEXT_RESPONSE_BYTES); + } + + /** + * 获取文件响应字节数上限。 + * + * @return 最大允许字节数 + */ + private long resolveMaxFileResponseBytes() { + return Long.getLong(MAX_FILE_RESPONSE_BYTES_PROPERTY, DEFAULT_MAX_FILE_RESPONSE_BYTES); + } + + /** + * 创建统一的响应超限异常。 + * + * @param maxResponseBytes 最大允许字节数 + * @return 响应超限异常 + */ + private IOException responseSizeExceeded(long maxResponseBytes) { + return new IOException("HTTP response body exceeds limit: " + maxResponseBytes + " bytes"); + } + public static class HttpServerErrorException extends IOException { private final int statusCode; @@ -340,6 +487,81 @@ public class HttpNode extends BaseNode { } } + /** + * 按实际读取量限制 HTTP 响应大小的输入流。 + */ + private final class LimitedResponseInputStream extends FilterInputStream { + + private final long maxBytes; + private long consumed; + + /** + * 创建受限响应流。 + * + * @param inputStream 原始输入流 + * @param maxBytes 最大允许字节数 + */ + private LimitedResponseInputStream(InputStream inputStream, long maxBytes) { + super(inputStream); + this.maxBytes = maxBytes; + } + + /** + * 读取单个字节并校验累计读取量。 + * + * @return 读取的字节或 {@code -1} + * @throws IOException 读取失败或超过限制时抛出 + */ + @Override + public int read() throws IOException { + int value = super.read(); + if (value >= 0) { + recordRead(1L); + } + return value; + } + + /** + * 批量读取并校验累计读取量。 + * + * @param buffer 目标缓冲区 + * @param offset 写入偏移 + * @param length 最大读取长度 + * @return 实际读取长度或 {@code -1} + * @throws IOException 读取失败或超过限制时抛出 + */ + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + long remainingWithProbe = maxBytes == Long.MAX_VALUE + ? Long.MAX_VALUE + : maxBytes - consumed + 1L; + int allowed = (int) Math.min( + Math.max(0L, remainingWithProbe), + (long) length); + if (allowed <= 0) { + throw responseSizeExceeded(maxBytes); + } + int count = super.read(buffer, offset, allowed); + if (count > 0) { + recordRead(count); + } + return count; + } + + /** + * 记录实际读取量。 + * + * @param count 本次读取字节数 + * @throws IOException 超过限制时抛出 + */ + private void recordRead(long count) throws IOException { + consumed += count; + if (consumed > maxBytes) { + throw responseSizeExceeded(maxBytes); + } + } + } + @Override public String toString() { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/KnowledgeNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/KnowledgeNode.java index a92bcfa..e834b46 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/KnowledgeNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/KnowledgeNode.java @@ -16,6 +16,7 @@ package com.easyagents.flow.core.node; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.knowledge.Knowledge; import com.easyagents.flow.core.knowledge.KnowledgeManager; import com.easyagents.flow.core.util.Maps; @@ -29,6 +30,8 @@ import java.util.List; import java.util.Map; public class KnowledgeNode extends BaseNode { + private static final long serialVersionUID = 1L; + private static final Logger logger = org.slf4j.LoggerFactory.getLogger(KnowledgeNode.class); @@ -71,11 +74,16 @@ public class KnowledgeNode extends BaseNode { @Override public Map execute(Chain chain) { - Map argsMap = chain.getState().resolveParameters(this); + ChainState chainState = + chain.getExecutionState(); + Map argsMap = + chainState.resolveParameters(this); + List> templateRootMaps = + chainState.buildTemplateRootMaps(argsMap); String realKeyword = TextTemplate.of(keyword) - .formatToString(chain.getState().buildTemplateRootMaps(argsMap)); + .formatToString(templateRootMaps); String realLimitString = TextTemplate.of(limit) - .formatToString(chain.getState().buildTemplateRootMaps(argsMap)); + .formatToString(templateRootMaps); int realLimit = 10; if (StringUtil.hasText(realLimitString)) { try { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java index 177a713..e62e31b 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java @@ -17,6 +17,7 @@ package com.easyagents.flow.core.node; import com.alibaba.fastjson.JSON; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.llm.Llm; import com.easyagents.flow.core.llm.LlmManager; @@ -26,6 +27,8 @@ import java.io.File; import java.util.*; public class LlmNode extends BaseNode { + private static final long serialVersionUID = 1L; + protected String llmId; protected Llm.ChatOptions chatOptions; @@ -88,14 +91,20 @@ public class LlmNode extends BaseNode { @Override public Map execute(Chain chain) { - Map parameterValues = chain.getState().resolveParameters(this); + ChainState chainState = + chain.getExecutionState(); + Map parameterValues = + chainState.resolveParameters(this); if (StringUtil.noText(userPrompt)) { throw new RuntimeException("Can not find user prompt"); } + List> templateRootMaps = + chainState.buildTemplateRootMaps( + parameterValues); String userPromptString = TextTemplate.of(userPrompt) - .formatToString(chain.getState().buildTemplateRootMaps(parameterValues)); + .formatToString(templateRootMaps); Llm llm = LlmManager.getInstance().getChatModel(this.llmId); @@ -104,14 +113,16 @@ public class LlmNode extends BaseNode { } String systemPromptString = TextTemplate.of(this.systemPrompt) - .formatToString(chain.getState().buildTemplateRootMaps(parameterValues)); + .formatToString(templateRootMaps); Llm.MessageInfo messageInfo = new Llm.MessageInfo(); messageInfo.setMessage(userPromptString); messageInfo.setSystemMessage(systemPromptString); if (images != null && !images.isEmpty()) { - Map filesMap = chain.getState().resolveParameters(this, images); + Map filesMap = + chainState.resolveParameters( + this, images); List imagesUrls = new ArrayList<>(); filesMap.forEach((s, o) -> { if (o instanceof String) { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java index f7c0ddc..ff7f2bf 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java @@ -18,19 +18,30 @@ package com.easyagents.flow.core.node; import com.easyagents.flow.core.chain.*; import com.easyagents.flow.core.chain.repository.ChainStateField; +import com.easyagents.flow.core.chain.repository.LoopInputReference; import com.easyagents.flow.core.chain.repository.NodeStateField; import com.easyagents.flow.core.chain.runtime.Trigger; import com.easyagents.flow.core.chain.runtime.TriggerContext; import com.easyagents.flow.core.chain.runtime.TriggerType; -import com.easyagents.flow.core.util.IterableUtil; +import com.easyagents.flow.core.chain.runtime.ExecutionBudgetExceededException; import com.easyagents.flow.core.util.Maps; import com.easyagents.flow.core.util.StringUtil; import java.io.Serializable; +import java.lang.reflect.Array; +import java.math.BigDecimal; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class LoopNode extends BaseNode { + private static final long serialVersionUID = 1L; + + private static final int DIRECT_INDEX_MAX_ITEMS = Math.max( + 1, + Integer.getInteger( + "tinyflow.loop.direct-index.max-items", 64)); private Parameter loopVar; @@ -44,6 +55,43 @@ public class LoopNode extends BaseNode { @Override public Map execute(Chain chain) { + MaterializationPlan[] planHolder = new MaterializationPlan[1]; + Map initialResult = chain.executeWithLock( + chain.getStateInstanceId(), + 10, + TimeUnit.SECONDS, + () -> executeLocked(chain, false, planHolder)); + MaterializationPlan plan = planHolder[0]; + if (plan == null) { + return initialResult; + } + + int iterableSize = chain.storeLoopInputOutsideLock( + plan.resultId, + plan.items, + materializationLimit(chain), + plan.claimId, + plan.claimGeneration); + return chain.executeWithLock( + chain.getStateInstanceId(), + 10, + TimeUnit.SECONDS, + () -> publishMaterializedInputAndContinue( + chain, plan, iterableSize)); + } + + /** + * 在实例锁内推进一次循环状态机。 + * + * @param chain 当前工作流 + * @param resumeAfterMaterialization 是否在本次调用中完成了锁外物化 + * @param planHolder 待锁外执行的物化计划容器 + * @return 节点执行结果 + */ + private Map executeLocked( + Chain chain, + boolean resumeAfterMaterialization, + MaterializationPlan[] planHolder) { Trigger prevTrigger = TriggerContext.getCurrentTrigger(); Deque loopStack = getOrCreateLoopStack(chain); @@ -51,15 +99,22 @@ public class LoopNode extends BaseNode { // 判断是否是首次进入该 LoopNode(即不是由子节点返回) TriggerType triggerType = prevTrigger.getType(); - boolean isFirstEntry = triggerType != TriggerType.PARENT && triggerType != TriggerType.SELF; + boolean isFirstEntry = !resumeAfterMaterialization + && triggerType != TriggerType.PARENT + && triggerType != TriggerType.SELF; if (isFirstEntry) { // 首次触发:创建新的 LoopContext 并压入堆栈 loopContext = new LoopContext(); loopContext.currentIndex = 0; - loopContext.subResult = new HashMap<>(); + loopContext.resultId = chain.getStateInstanceId() + ":" + UUID.randomUUID(); // 保存原始触发上下文(用于循环结束后恢复) loopStack.offerLast(loopContext); + int nestedDepth = chain.getNestedDepthBase() + + (prevTrigger == null + ? 1 + : prevTrigger.getLoopCursors().size() + 1); + chain.getExecutionBudget().checkNestedDepth(this.id, nestedDepth); chain.updateNodeStateSafely(this.id, state -> { state.getMemory().put(buildLoopStackId(), loopStack); @@ -83,33 +138,114 @@ public class LoopNode extends BaseNode { loopContext = loopStack.peekFirst(); } + if (loopContext.materializingInput && !resumeAfterMaterialization) { + String currentClaimId = chain.currentFencingClaimId(); + long currentGeneration = chain.currentClaimGeneration(); + if (Objects.equals(loopContext.materializationClaimId, currentClaimId) + && loopContext.materializationClaimGeneration == currentGeneration) { + return waitingResult(); + } + // 原 owner 已失去 claim,由新代际使用全新 resultId 接管。 + loopContext.resultId = + chain.getStateInstanceId() + ":" + UUID.randomUUID(); + loopContext.materializationClaimId = currentClaimId; + loopContext.materializationClaimGeneration = currentGeneration; + persistLoopStack(chain, loopStack); + } -// LoopContext loopContext = getLoopContext(prevTrigger, chain); -// int triggerLoopIndex = getTriggerLoopIndex(prevTrigger); -// -// if (loopContext.currentIndex != triggerLoopIndex) { -// // 不执行,子流程有分叉,已经被其他的分叉节点触发了 -// return Maps.of(ChainConsts.SCHEDULE_NEXT_NODE_DISABLED_KEY, true) -// .set(ChainConsts.NODE_STATE_STATUS_KEY, NodeStatus.RUNNING); -// } - - Map loopVars = chain.getState().resolveParameters(this, Collections.singletonList(loopVar)); - Object loopValue = loopVars.get(loopVar.getName()); + migrateLegacyResult(chain, loopContext); + if (!acceptParentBranch(prevTrigger, loopContext, chain, loopStack)) { + return Maps.of(ChainConsts.SCHEDULE_NEXT_NODE_DISABLED_KEY, true) + .set(ChainConsts.NODE_STATE_STATUS_KEY, NodeStatus.RUNNING); + } int shouldLoopCount; - if (loopValue instanceof Iterable) { - shouldLoopCount = IterableUtil.size((Iterable) loopValue); - } else if (loopValue instanceof Number || (loopValue instanceof String && StringUtil.isNumeric(loopValue.toString()))) { - shouldLoopCount = loopValue instanceof Number ? ((Number) loopValue).intValue() : Integer.parseInt(loopValue.toString().trim()); + boolean storedIterable = false; + boolean directlyIndexed = false; + boolean numericLoop = false; + Object loopValue = null; + if (loopContext.iterableInputStored) { + // 已物化输入不再解析原始参数,避免大集合在每轮反序列化和路径解析。 + shouldLoopCount = loopContext.iterableSize; + storedIterable = true; } else { - throw new IllegalArgumentException("loopValue must be Iterable or Number or String, but loopValue is \"" + loopValue + "\""); + LoopInputReference storedInput = + resolveStoredInputReference(chain); + if (storedInput != null) { + loopContext.resultId = + storedInput.getResultId(); + loopContext.iterableSize = + storedInput.getItemCount(); + loopContext.iterableInputStored = true; + loopContext.inputExternalized = true; + shouldLoopCount = + storedInput.getItemCount(); + storedIterable = true; + persistLoopStack(chain, loopStack); + } else { + Map loopVars = + chain.getExecutionState().resolveParameters( + this, + Collections.singletonList( + loopVar)); + loopValue = loopVars.get(loopVar.getName()); + Iterable iterableInput = + toIterableInput(loopValue); + if (iterableInput != null) { + int knownSize = knownInputSize(loopValue); + if (knownSize >= 0) { + checkExplicitLoopIterations(chain, knownSize, true); + } + if (knownSize >= 0 + && knownSize <= DIRECT_INDEX_MAX_ITEMS) { + shouldLoopCount = knownSize; + directlyIndexed = true; + } else { + /* + * 每次未完成的物化尝试使用新结果 ID。失锁 owner 的旧分块仅会按 TTL + * 回收,接管者不会复用或删除其部分数据。 + */ + loopContext.resultId = + chain.getStateInstanceId() + ":" + UUID.randomUUID(); + loopContext.materializingInput = true; + loopContext.materializationClaimId = + chain.currentFencingClaimId(); + loopContext.materializationClaimGeneration = + chain.currentClaimGeneration(); + persistLoopStack(chain, loopStack); + if (planHolder == null) { + throw new IllegalStateException( + "Loop materialization plan holder is unavailable"); + } + planHolder[0] = new MaterializationPlan( + loopContext.resultId, + iterableInput, + loopContext.materializationClaimId, + loopContext.materializationClaimGeneration); + return waitingResult(); + } + } else if (loopValue instanceof Number + || loopValue instanceof String) { + shouldLoopCount = parseNumericLoopCount(loopValue); + numericLoop = true; + } else { + throw invalidLoopValue(loopValue); + } + } } + checkExplicitLoopIterations(chain, shouldLoopCount, !numericLoop); // 不是第一次执行,合并结果到 subResult if (loopContext.currentIndex != 0) { - ChainState subState = chain.getState(); + ChainState subState = + chain.getExecutionState(); Map currentOutputs = collectCurrentOutputValues(subState); - mergeResult(loopContext.subResult, currentOutputs); + loopContext.accumulatedBytes += estimateBytes(currentOutputs, new IdentityHashMap<>()); + chain.getExecutionBudget().checkAccumulatedBytes(this.id, loopContext.accumulatedBytes); + chain.appendLoopResult( + loopContext.resultId, + loopContext.currentIndex - 1, + currentOutputs); // 将上一轮最新输出同步到循环节点作用域,供下一轮循环体读取。 publishLoopProgress(chain, currentOutputs); } @@ -128,25 +264,37 @@ public class LoopNode extends BaseNode { if (!loopStack.isEmpty()) { chain.scheduleNode(this, null, TriggerType.SELF, 0); } - return loopContext.subResult; + if (prevTrigger != null) { + prevTrigger.getLoopCursors().remove(this.id); + } + Map completedResult = chain.getLoopResultRepository().references( + loopContext.resultId, loopContext.currentIndex, getOutputNames()); + chain.getLoopResultRepository().releaseActiveCache( + loopContext.resultId); + if (!loopContext.inputExternalized) { + chain.removeLoopInput(loopContext.resultId); + } + return completedResult; } int loopIndex = loopContext.currentIndex; loopContext.currentIndex++; - chain.updateNodeStateSafely(this.id, state -> { - state.getMemory().put(buildLoopStackId(), loopStack); - return EnumSet.of(NodeStateField.MEMORY); - }); + persistLoopStack(chain, loopStack); - if (loopValue instanceof Iterable) { - Object loopItem = IterableUtil.get((Iterable) loopValue, loopIndex); + if (storedIterable) { + Object loopItem = chain.getLoopResultRepository().loadInputItem(loopContext.resultId, loopIndex); executeLoopChain(chain, loopContext, loopItem); - } else if (loopValue instanceof Number || (loopValue instanceof String && StringUtil.isNumeric(loopValue.toString()))) { + } else if (directlyIndexed) { + executeLoopChain( + chain, + loopContext, + directInputItem(loopValue, loopIndex)); + } else if (numericLoop) { executeLoopChain(chain, loopContext, loopIndex); } else { - throw new IllegalArgumentException("loopValue must be Iterable or Number or String, but loopValue is \"" + loopValue + "\""); + throw invalidLoopValue(loopValue); } // 禁用调度下个节点 @@ -154,6 +302,281 @@ public class LoopNode extends BaseNode { .set(ChainConsts.NODE_STATE_STATUS_KEY, NodeStatus.RUNNING); } + /** + * 在短实例锁内发布锁外物化结果并继续循环状态机。 + * + * @param chain 当前工作流 + * @param plan 已完成的物化计划 + * @param iterableSize 物化元素数 + * @return 本轮循环执行结果 + */ + private Map publishMaterializedInputAndContinue( + Chain chain, MaterializationPlan plan, int iterableSize) { + if (!chain.isExecutionActiveNow()) { + throw new com.easyagents.flow.core.chain.runtime.TriggerClaimLostException( + "loop-materialization-cancelled:" + plan.resultId); + } + Deque loopStack = getOrCreateLoopStack(chain); + LoopContext context = loopStack.peekFirst(); + if (context == null + || !context.materializingInput + || !Objects.equals(context.resultId, plan.resultId) + || !Objects.equals( + context.materializationClaimId, plan.claimId) + || context.materializationClaimGeneration + != plan.claimGeneration) { + throw new com.easyagents.flow.core.chain.runtime.TriggerClaimLostException( + "loop-materialization:" + plan.resultId); + } + context.iterableSize = iterableSize; + context.iterableInputStored = true; + context.materializingInput = false; + context.inputExternalized = externalizeMaterializedInput( + chain, plan.resultId, iterableSize); + persistLoopStack(chain, loopStack); + return executeLocked(chain, true, null); + } + + /** + * 将热状态中的大型原始输入替换为轻量引用。 + * + *

仅替换参数直接对应的扁平内存键;无法准确定位的嵌套路径保持原值, + * 以业务兼容性优先。其他节点读取该引用时由仓储透明还原完整列表。

+ * + * @param chain 当前工作流 + * @param resultId 已物化输入 ID + * @param iterableSize 输入元素数 + * @return 已替换热状态值时为 {@code true} + */ + private boolean externalizeMaterializedInput( + Chain chain, String resultId, int iterableSize) { + String ref = loopVar == null ? null : loopVar.getRef(); + String name = loopVar == null ? null : loopVar.getName(); + AtomicBoolean replaced = new AtomicBoolean(); + chain.updateStateSafely(state -> { + ConcurrentHashMap memory = + state.getMemory(); + String key = StringUtil.hasText(ref) + && memory.containsKey(ref) + ? ref + : (StringUtil.hasText(name) + && memory.containsKey(name) + ? name + : null); + if (key == null) { + return null; + } + Object current = memory.get(key); + if (current instanceof LoopInputReference) { + replaced.set(true); + return null; + } + memory.put( + key, + new LoopInputReference( + resultId, iterableSize)); + replaced.set(true); + return EnumSet.of(ChainStateField.MEMORY); + }); + return replaced.get(); + } + + /** + * 直接识别上游已分页物化的输入引用,避免参数解析边界先还原完整列表。 + * + * @param chain 当前工作流 + * @return 已物化输入引用;不存在时为 {@code null} + */ + private LoopInputReference resolveStoredInputReference( + Chain chain) { + if (loopVar == null) { + return null; + } + Map memory = + chain.getExecutionState().getMemory(); + String ref = loopVar.getRef(); + if (StringUtil.hasText(ref) + && memory.get(ref) + instanceof LoopInputReference) { + return (LoopInputReference) memory.get(ref); + } + String name = loopVar.getName(); + return StringUtil.hasText(name) + && memory.get(name) + instanceof LoopInputReference + ? (LoopInputReference) memory.get(name) + : null; + } + + /** + * 构造不推进下游的循环等待结果。 + * + * @return 运行态控制结果 + */ + private Map waitingResult() { + return Maps.of(ChainConsts.SCHEDULE_NEXT_NODE_DISABLED_KEY, true) + .set(ChainConsts.NODE_STATE_STATUS_KEY, NodeStatus.RUNNING); + } + + /** + * 将集合或数组统一转换为单次消费的 Iterable。 + * + * @param loopValue 循环输入 + * @return 可迭代输入;数值循环返回 {@code null} + */ + private Iterable toIterableInput(Object loopValue) { + if (loopValue instanceof Iterable) { + return (Iterable) loopValue; + } + if (loopValue != null && loopValue.getClass().isArray()) { + int length = Array.getLength(loopValue); + return () -> new Iterator() { + private int index; + + @Override + public boolean hasNext() { + return index < length; + } + + @Override + public Object next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return Array.get(loopValue, index++); + } + }; + } + return null; + } + + /** + * 获取无需遍历即可确定的输入元素数量。 + * + * @param loopValue 循环输入 + * @return 已知数量;未知时返回 {@code -1} + */ + private int knownInputSize(Object loopValue) { + if (loopValue instanceof Collection) { + return ((Collection) loopValue).size(); + } + if (loopValue != null && loopValue.getClass().isArray()) { + return Array.getLength(loopValue); + } + return -1; + } + + /** + * 解析数值型循环次数,循环索引从 0 开始并执行到 {@code count - 1}。 + * + * @param loopValue 数值或数值字符串 + * @return 1~300 的循环总次数 + * @throws IllegalArgumentException 输入不是范围内整数 + */ + private int parseNumericLoopCount(Object loopValue) { + final int count; + try { + count = new BigDecimal(String.valueOf(loopValue).trim()) + .intValueExact(); + } catch (ArithmeticException | NumberFormatException exception) { + throw new IllegalArgumentException( + "Loop count must be an integer between " + + Node.MIN_LOOP_COUNT + + " and " + + Node.MAX_LOOP_COUNT, + exception); + } + if (count < Node.MIN_LOOP_COUNT || count > Node.MAX_LOOP_COUNT) { + throw new IllegalArgumentException( + "Loop count must be between " + + Node.MIN_LOOP_COUNT + + " and " + + Node.MAX_LOOP_COUNT + + ", but was " + + count); + } + return count; + } + + /** + * 校验显式循环节点的本层迭代次数。 + * + * @param chain 当前工作流 + * @param iterations 本层计划迭代次数 + * @param allowEmpty 是否允许空集合产生零次迭代 + * @throws IllegalArgumentException 不允许的负数或零次数 + * @throws ExecutionBudgetExceededException 超过 300 次或部署预算 + */ + private void checkExplicitLoopIterations( + Chain chain, long iterations, boolean allowEmpty) { + if (iterations < 0L || (!allowEmpty && iterations == 0L)) { + throw new IllegalArgumentException( + "Loop count must be at least " + Node.MIN_LOOP_COUNT); + } + if (iterations > Node.MAX_LOOP_COUNT) { + throw new ExecutionBudgetExceededException( + "Loop iteration limit exceeded for node " + + this.id + + ": " + + iterations + + " > " + + Node.MAX_LOOP_COUNT); + } + chain.getExecutionBudget().checkIterations(this.id, iterations); + } + + /** + * 计算未知 Iterable 的物化上限,同时服从平台预算和 300 次硬上限。 + * + * @param chain 当前工作流 + * @return 正数物化上限 + */ + private long materializationLimit(Chain chain) { + long budgetLimit = chain.getExecutionBudget().getMaxIterations(); + if (budgetLimit <= 0L) { + return Node.MAX_LOOP_COUNT; + } + return Math.min(budgetLimit, Node.MAX_LOOP_COUNT); + } + + /** + * 按序号读取小型集合或数组,避免额外 Redis 分块读写。 + * + * @param loopValue 小型循环输入 + * @param index 元素序号 + * @return 对应元素 + */ + private Object directInputItem(Object loopValue, int index) { + if (loopValue instanceof List) { + return ((List) loopValue).get(index); + } + if (loopValue != null && loopValue.getClass().isArray()) { + return Array.get(loopValue, index); + } + if (loopValue instanceof Collection) { + Iterator iterator = + ((Collection) loopValue).iterator(); + for (int current = 0; current < index; current++) { + iterator.next(); + } + return iterator.next(); + } + throw invalidLoopValue(loopValue); + } + + /** + * 创建循环输入类型错误。 + * + * @param loopValue 非法输入 + * @return 参数异常 + */ + private IllegalArgumentException invalidLoopValue(Object loopValue) { + return new IllegalArgumentException( + "loopValue must be Iterable, array, Number or String, but loopValue is \"" + + loopValue + + "\""); + } + /** * 获取或创建当前节点的 LoopContext 堆栈(每个 LoopNode 实例独立) @@ -168,15 +591,10 @@ public class LoopNode extends BaseNode { stack = (Deque) stackObj; } else { stack = new ArrayDeque<>(); - chain.updateNodeStateSafely(this.id, state -> { - state.getMemory().put(key, stack); - return EnumSet.of(NodeStateField.MEMORY); - }); } return stack; } - private void executeLoopChain(Chain chain, LoopContext loopContext, Object loopItem) { chain.updateStateSafely(state -> { @@ -188,13 +606,116 @@ public class LoopNode extends BaseNode { ChainDefinition definition = chain.getDefinition(); - List outwardEdges = definition.getOutwardEdge(this.id); - for (Edge edge : outwardEdges) { - Node childNode = definition.getNodeById(edge.getTarget()); - if (childNode.getParentId() != null && childNode.getParentId().equals(this.id)) { - chain.scheduleNode(childNode, edge.getId(), TriggerType.CHILD, 0); + List childDispatches = + definition.getLoopChildDispatches(this.id); + if (childDispatches.isEmpty()) { + throw new IllegalStateException("Loop node has no executable child branch: " + this.id); + } + loopContext.expectedReturnCount = childDispatches.size(); + loopContext.getCompletedBranchIds().clear(); + for (ChainDefinition.LoopChildDispatch dispatch : + childDispatches) { + Trigger.LoopCursor cursor = new Trigger.LoopCursor( + loopContext.resultId, + loopContext.currentIndex - 1, + dispatch.getBranchId()); + chain.scheduleLoopChild( + dispatch.getNode(), + dispatch.getEdgeId(), + 0, + this.id, + cursor); + } + } + + /** + * 校验父分支回调的代际和分支屏障。 + * + * @param trigger 当前触发器 + * @param context 当前循环上下文 + * @param chain 当前工作流 + * @param loopStack 循环上下文栈 + * @return 当前回调是否为本轮最后一个有效分支 + */ + private boolean acceptParentBranch(Trigger trigger, + LoopContext context, + Chain chain, + Deque loopStack) { + if (trigger == null || trigger.getType() != TriggerType.PARENT) { + return true; + } + Trigger.LoopCursor cursor = trigger.getLoopCursors().get(this.id); + if (cursor == null) { + return true; + } + if (!Objects.equals(context.resultId, cursor.getResultId())) { + return false; + } + int expectedIndex = context.currentIndex - 1; + if (cursor.getIterationIndex() < expectedIndex) { + return false; + } + if (cursor.getIterationIndex() > expectedIndex) { + throw new IllegalStateException("Future loop callback index: " + cursor.getIterationIndex()); + } + if (!context.getCompletedBranchIds().add(cursor.getBranchId())) { + return false; + } + persistLoopStack(chain, loopStack); + if (context.getCompletedBranchIds().size() < Math.max(1, context.expectedReturnCount)) { + return false; + } + context.getCompletedBranchIds().clear(); + return true; + } + + /** + * 将旧版本内嵌累计结果惰性迁移到分块仓储。 + * + * @param chain 当前工作流 + * @param context 循环上下文 + */ + @SuppressWarnings("unchecked") + private void migrateLegacyResult(Chain chain, LoopContext context) { + if (context.resultId != null) { + return; + } + context.resultId = chain.getStateInstanceId() + ":" + UUID.randomUUID(); + if (context.subResult != null && !context.subResult.isEmpty()) { + int migratedCount = 0; + for (Object value : context.subResult.values()) { + if (value instanceof List) { + migratedCount = Math.max(migratedCount, ((List) value).size()); + } + } + for (int index = 0; index < migratedCount; index++) { + Map outputs = new LinkedHashMap<>(); + for (Map.Entry entry : context.subResult.entrySet()) { + if (entry.getValue() instanceof List + && index < ((List) entry.getValue()).size()) { + outputs.put(entry.getKey(), ((List) entry.getValue()).get(index)); + } + } + chain.appendLoopResult( + context.resultId, + index, + outputs); } } + context.subResult = null; + } + + /** + * 保存紧凑循环上下文。 + * + * @param chain 当前工作流 + * @param loopStack 循环上下文栈 + */ + private void persistLoopStack(Chain chain, Deque loopStack) { + chain.updateNodeStateSafely(this.id, state -> { + state.getMemory().put(buildLoopStackId(), loopStack); + return EnumSet.of(NodeStateField.MEMORY); + }); } @@ -227,20 +748,69 @@ public class LoopNode extends BaseNode { } - private void mergeResult(Map toResult, Map currentOutputs) { + /** + * 获取循环输出名称并保持定义顺序。 + * + * @return 输出名称列表 + */ + private List getOutputNames() { + List outputNames = new ArrayList<>(); List outputDefs = getOutputDefs(); if (outputDefs != null) { for (Parameter outputDef : outputDefs) { - Object value = currentOutputs.get(outputDef.getName()); - - @SuppressWarnings("unchecked") List existList = (List) toResult.get(outputDef.getName()); - if (existList == null) { - existList = new ArrayList<>(); - } - existList.add(value); - toResult.put(outputDef.getName(), existList); + outputNames.add(outputDef.getName()); } } + return outputNames; + } + + /** + * 估算本轮输出占用字节数,用于宽松的累计结果失控保护。 + * + * @param value 待估算值 + * @param visited 已访问对象集合,防止循环引用 + * @return 估算字节数 + */ + private long estimateBytes(Object value, IdentityHashMap visited) { + if (value == null) { + return 0L; + } + if (value instanceof String) { + return (long) ((String) value).length() * Character.BYTES; + } + if (value instanceof byte[]) { + return ((byte[]) value).length; + } + if (value instanceof Number || value instanceof Date) { + return 16L; + } + if (value instanceof Boolean || value instanceof Character) { + return 2L; + } + if (visited.put(value, Boolean.TRUE) != null) { + return 0L; + } + long bytes = 0L; + if (value instanceof Map) { + for (Map.Entry entry : ((Map) value).entrySet()) { + bytes += estimateBytes(entry.getKey(), visited); + bytes += estimateBytes(entry.getValue(), visited); + } + } else if (value instanceof Collection) { + for (Object item : (Collection) value) { + bytes += estimateBytes(item, visited); + } + } else if (value instanceof Iterable) { + bytes = 64L; + } else if (value.getClass().isArray()) { + int length = Array.getLength(value); + for (int index = 0; index < length; index++) { + bytes += estimateBytes(Array.get(value, index), visited); + } + } else { + bytes = 64L; + } + return bytes; } @@ -274,8 +844,20 @@ public class LoopNode extends BaseNode { public static class LoopContext implements Serializable { + private static final long serialVersionUID = 5258356831772776243L; + int currentIndex; + String resultId; Map subResult; + boolean iterableInputStored; + boolean inputExternalized; + boolean materializingInput; + String materializationClaimId; + long materializationClaimGeneration; + int iterableSize; + long accumulatedBytes; + int expectedReturnCount = 1; + Set completedBranchIds = new LinkedHashSet<>(); public int getCurrentIndex() { return currentIndex; @@ -285,6 +867,14 @@ public class LoopNode extends BaseNode { this.currentIndex = currentIndex; } + public String getResultId() { + return resultId; + } + + public void setResultId(String resultId) { + this.resultId = resultId; + } + public Map getSubResult() { return subResult; } @@ -293,5 +883,111 @@ public class LoopNode extends BaseNode { this.subResult = subResult; } + public long getAccumulatedBytes() { + return accumulatedBytes; + } + + public void setAccumulatedBytes(long accumulatedBytes) { + this.accumulatedBytes = accumulatedBytes; + } + + public boolean isIterableInputStored() { + return iterableInputStored; + } + + public void setIterableInputStored(boolean iterableInputStored) { + this.iterableInputStored = iterableInputStored; + } + + /** + * @return 原始输入是否已从热状态替换为轻量引用 + */ + public boolean isInputExternalized() { + return inputExternalized; + } + + /** + * 设置原始输入外置状态。 + * + * @param inputExternalized 是否已替换为轻量引用 + */ + public void setInputExternalized( + boolean inputExternalized) { + this.inputExternalized = inputExternalized; + } + + /** + * @return 是否正在锁外物化输入 + */ + public boolean isMaterializingInput() { + return materializingInput; + } + + /** + * 设置锁外物化状态。 + * + * @param materializingInput 是否正在物化 + */ + public void setMaterializingInput(boolean materializingInput) { + this.materializingInput = materializingInput; + } + + public int getIterableSize() { + return iterableSize; + } + + public void setIterableSize(int iterableSize) { + this.iterableSize = iterableSize; + } + + public int getExpectedReturnCount() { + return expectedReturnCount; + } + + public void setExpectedReturnCount(int expectedReturnCount) { + this.expectedReturnCount = expectedReturnCount; + } + + public Set getCompletedBranchIds() { + if (completedBranchIds == null) { + completedBranchIds = new LinkedHashSet<>(); + } + return completedBranchIds; + } + + public void setCompletedBranchIds(Set completedBranchIds) { + this.completedBranchIds = completedBranchIds; + } + + } + + /** + * 一次锁外循环输入物化所需的不可变守卫快照。 + */ + private static final class MaterializationPlan { + + private final String resultId; + private final Iterable items; + private final String claimId; + private final long claimGeneration; + + /** + * 创建物化计划。 + * + * @param resultId 唯一结果 ID + * @param items 输入元素 + * @param claimId 触发器 ID + * @param claimGeneration 触发器代际 + */ + private MaterializationPlan( + String resultId, + Iterable items, + String claimId, + long claimGeneration) { + this.resultId = resultId; + this.items = items; + this.claimId = claimId; + this.claimGeneration = claimGeneration; + } } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/SearchEngineNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/SearchEngineNode.java index a3ed2cf..1246fb2 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/SearchEngineNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/SearchEngineNode.java @@ -16,6 +16,7 @@ package com.easyagents.flow.core.node; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.searchengine.SearchEngine; import com.easyagents.flow.core.searchengine.SearchEngineManager; import com.easyagents.flow.core.util.Maps; @@ -29,6 +30,8 @@ import java.util.List; import java.util.Map; public class SearchEngineNode extends BaseNode { + private static final long serialVersionUID = 1L; + private static final Logger logger = org.slf4j.LoggerFactory.getLogger(SearchEngineNode.class); @@ -62,11 +65,16 @@ public class SearchEngineNode extends BaseNode { @Override public Map execute(Chain chain) { - Map argsMap = chain.getState().resolveParameters(this); + ChainState chainState = + chain.getExecutionState(); + Map argsMap = + chainState.resolveParameters(this); + List> templateRootMaps = + chainState.buildTemplateRootMaps(argsMap); String realKeyword = TextTemplate.of(keyword) - .formatToString(chain.getState().buildTemplateRootMaps(argsMap)); + .formatToString(templateRootMaps); String realLimitString = TextTemplate.of(limit) - .formatToString(chain.getState().buildTemplateRootMaps(argsMap)); + .formatToString(templateRootMaps); int realLimit = 10; if (StringUtil.hasText(realLimitString)) { try { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/StartNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/StartNode.java index 566ca31..298fd1c 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/StartNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/StartNode.java @@ -21,9 +21,12 @@ import com.easyagents.flow.core.chain.Chain; import java.util.Map; public class StartNode extends BaseNode { + private static final long serialVersionUID = 1L; + @Override public Map execute(Chain chain) { - return chain.getState().resolveParameters(this); + return chain.getExecutionState() + .resolveParameters(this); } @Override diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java index 3709cac..d6006cb 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; public class TemplateNode extends BaseNode { + private static final long serialVersionUID = 1L; + private static final Engine engine; private String template; @@ -48,7 +50,8 @@ public class TemplateNode extends BaseNode { @Override public Map execute(Chain chain) { - Map parameters = chain.getState().resolveParameters(this); + Map parameters = + chain.getExecutionState().resolveParameters(this); ByteArrayOutputStream result = new ByteArrayOutputStream(); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java index 2ca2c8a..046bb32 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java @@ -20,11 +20,13 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.easyagents.flow.core.chain.DataType; import com.easyagents.flow.core.chain.JsCodeCondition; +import com.easyagents.flow.core.chain.Node; import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.chain.RefType; import com.easyagents.flow.core.node.BaseNode; import com.easyagents.flow.core.util.StringUtil; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -156,10 +158,7 @@ public abstract class BaseNodeParser implements NodeParser implements NodeParser targetSemaphores = new ConcurrentHashMap<>(); + private final Object targetRegistryLock = new Object(); + private final Semaphore overflowTargetSemaphore; + private final LongAdder acquiredCount = new LongAdder(); + private final LongAdder rejectedCount = new LongAdder(); + private final LongAdder inFlightCount = new LongAdder(); + private final LongAdder totalWaitNanos = new LongAdder(); + private final AtomicLong lastRejectionLogNanos = new AtomicLong(); + + /** + * 创建 I/O 隔离器。 + * + * @param globalConcurrency 当前进程允许的 I/O 总并发 + * @param perTargetConcurrency 单个目标允许的 I/O 并发 + * @param acquireTimeout 等待许可的最长时间 + * @param maxTrackedTargets 最多单独跟踪的目标数量 + * @throws IllegalArgumentException 参数不合法时抛出 + */ + public IoBulkhead( + int globalConcurrency, + int perTargetConcurrency, + Duration acquireTimeout, + int maxTrackedTargets) { + if (globalConcurrency <= 0 + || perTargetConcurrency <= 0 + || acquireTimeout == null + || acquireTimeout.isNegative() + || maxTrackedTargets <= 0) { + throw new IllegalArgumentException("I/O bulkhead configuration must be positive"); + } + this.globalSemaphore = new Semaphore(globalConcurrency, true); + this.perTargetConcurrency = perTargetConcurrency; + this.acquireTimeoutNanos = acquireTimeout.toNanos(); + this.maxTrackedTargets = maxTrackedTargets; + this.overflowTargetSemaphore = new Semaphore(perTargetConcurrency, true); + } + + /** + * 获取进程内共享 I/O 隔离器。 + * + * @return 共享隔离器 + */ + public static IoBulkhead shared() { + return SHARED; + } + + /** + * 获取数据集读写独立隔离器。 + * + * @return 数据集隔离器 + */ + public static IoBulkhead dataset() { + return DATASET; + } + + /** + * 获取对象存储读写独立隔离器。 + * + * @return 对象存储隔离器 + */ + public static IoBulkhead storage() { + return STORAGE; + } + + /** + * 获取文档解析独立隔离器。 + * + * @return 文档解析隔离器 + */ + public static IoBulkhead documentParse() { + return DOCUMENT_PARSE; + } + + /** + * 获取必须完整物化响应的独立小并发隔离器。 + * + * @return 响应聚合隔离器 + */ + public static IoBulkhead responseAggregation() { + return RESPONSE_AGGREGATION; + } + + /** + * 原子替换全部工作流 I/O 隔离器配置。 + * + *

应用应在开始处理工作流前调用。既有许可继续由旧实例释放, + * 后续请求读取新实例,不会中断正在执行的 I/O。

+ * + * @param http HTTP 请求配置 + * @param dataset 数据集配置 + * @param storage 对象存储配置 + * @param documentParse 文档解析配置 + * @param responseAggregation 响应聚合配置 + */ + public static synchronized void configure( + Settings http, + Settings dataset, + Settings storage, + Settings documentParse, + Settings responseAggregation) { + SHARED = create(http); + DATASET = create(dataset); + STORAGE = create(storage); + DOCUMENT_PARSE = create(documentParse); + RESPONSE_AGGREGATION = + create(responseAggregation); + } + + /** + * 根据不可变配置创建隔离器。 + * + * @param settings 隔离配置 + * @return 新隔离器 + */ + private static IoBulkhead create(Settings settings) { + Settings value = java.util.Objects.requireNonNull( + settings, "I/O bulkhead settings required"); + return new IoBulkhead( + value.globalConcurrency(), + value.perTargetConcurrency(), + value.acquireTimeout(), + value.maxTrackedTargets()); + } + + /** + * I/O 隔离器的不可变配置。 + * + * @param globalConcurrency 当前进程允许的 I/O 总并发 + * @param perTargetConcurrency 单个目标允许的 I/O 并发 + * @param acquireTimeout 等待许可的最长时间 + * @param maxTrackedTargets 最多单独跟踪的目标数量 + */ + public record Settings( + int globalConcurrency, + int perTargetConcurrency, + Duration acquireTimeout, + int maxTrackedTargets) { + + /** + * 校验配置,避免应用启动后才暴露无效容量。 + * + * @throws IllegalArgumentException 配置值无效时抛出 + */ + public Settings { + if (globalConcurrency <= 0 + || perTargetConcurrency <= 0 + || acquireTimeout == null + || acquireTimeout.isNegative() + || acquireTimeout.isZero() + || maxTrackedTargets <= 0) { + throw new IllegalArgumentException( + "I/O bulkhead settings must be positive"); + } + } + } + + /** + * 将 URL 转换为稳定的协议和主机目标键。 + * + * @param url 请求 URL + * @return 目标键;URL 不合法时返回通用 HTTP 目标 + */ + public static String targetForUrl(String url) { + if (!StringUtil.hasText(url)) { + return "http:unknown"; + } + try { + URI uri = URI.create(url.trim()); + String host = uri.getHost(); + if (!StringUtil.hasText(host)) { + return "http:unknown"; + } + int port = uri.getPort(); + return "http:" + + host.toLowerCase(Locale.ROOT) + + (port < 0 ? "" : ":" + port); + } catch (IllegalArgumentException exception) { + return "http:unknown"; + } + } + + /** + * 在给定目标上申请一次阻塞 I/O 许可。 + * + * @param target 目标标识 + * @return 使用完成后必须关闭的许可 + * @throws RetryableTriggerException 等待超时或线程被中断时抛出 + */ + public Permit acquire(String target) { + String normalizedTarget = normalizeTarget(target); + Semaphore targetSemaphore = targetSemaphore(normalizedTarget); + long startedAt = System.nanoTime(); + boolean targetAcquired = false; + boolean globalAcquired = false; + try { + targetAcquired = targetSemaphore.tryAcquire(acquireTimeoutNanos, TimeUnit.NANOSECONDS); + if (!targetAcquired) { + throw rejection(normalizedTarget, startedAt, null); + } + long remainingNanos = Math.max( + 0L, + acquireTimeoutNanos - (System.nanoTime() - startedAt)); + globalAcquired = globalSemaphore.tryAcquire(remainingNanos, TimeUnit.NANOSECONDS); + if (!globalAcquired) { + throw rejection(normalizedTarget, startedAt, null); + } + totalWaitNanos.add(System.nanoTime() - startedAt); + acquiredCount.increment(); + inFlightCount.increment(); + return new Permit(this, targetSemaphore); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw rejection(normalizedTarget, startedAt, exception); + } finally { + if (!globalAcquired && targetAcquired) { + targetSemaphore.release(); + } + } + } + + /** + * 获取当前隔离器的轻量运行指标。 + * + * @return 指标快照 + */ + public Snapshot snapshot() { + return new Snapshot( + acquiredCount.sum(), + rejectedCount.sum(), + inFlightCount.sum(), + totalWaitNanos.sum(), + globalSemaphore.availablePermits(), + targetSemaphores.size()); + } + + /** + * 释放成功申请的全局和目标许可。 + * + * @param targetSemaphore 已申请的目标信号量 + */ + private void release(Semaphore targetSemaphore) { + inFlightCount.decrement(); + globalSemaphore.release(); + targetSemaphore.release(); + } + + /** + * 获取目标对应的信号量,并限制目标注册表体积。 + * + * @param target 规范化目标 + * @return 目标信号量 + */ + private Semaphore targetSemaphore(String target) { + Semaphore existing = targetSemaphores.get(target); + if (existing != null) { + return existing; + } + synchronized (targetRegistryLock) { + existing = targetSemaphores.get(target); + if (existing != null) { + return existing; + } + if (targetSemaphores.size() >= maxTrackedTargets) { + return overflowTargetSemaphore; + } + Semaphore created = new Semaphore(perTargetConcurrency, true); + targetSemaphores.put(target, created); + return created; + } + } + + /** + * 构造可重新投递的限流异常并记录指标。 + * + * @param target 目标标识 + * @param startedAt 等待开始时间 + * @param cause 原始异常 + * @return 可重新投递异常 + */ + private RetryableTriggerException rejection(String target, long startedAt, Throwable cause) { + long waitedNanos = System.nanoTime() - startedAt; + totalWaitNanos.add(waitedNanos); + rejectedCount.increment(); + TimeoutException timeoutException = new TimeoutException( + "I/O bulkhead is saturated for target " + target); + if (cause != null) { + timeoutException.initCause(cause); + } + logRejectionIfDue(target, waitedNanos); + return new RetryableTriggerException("I/O 资源繁忙,请稍后重试", timeoutException); + } + + /** + * 对饱和日志限频,避免过载期间放大日志 I/O。 + * + * @param target 目标标识 + * @param waitedNanos 本次等待纳秒数 + */ + private void logRejectionIfDue(String target, long waitedNanos) { + long now = System.nanoTime(); + long previous = lastRejectionLogNanos.get(); + if ((previous != 0L && now - previous < REJECTION_LOG_INTERVAL_NANOS) + || !lastRejectionLogNanos.compareAndSet(previous, now)) { + return; + } + log.warn( + "工作流 I/O 隔离器拒绝请求,target={}, waitedMs={}, inFlight={}, rejected={}", + target, + TimeUnit.NANOSECONDS.toMillis(waitedNanos), + inFlightCount.sum(), + rejectedCount.sum()); + } + + /** + * 规范化目标键。 + * + * @param target 原始目标 + * @return 非空目标键 + */ + private static String normalizeTarget(String target) { + return StringUtil.hasText(target) ? target.trim() : "io:unknown"; + } + + /** + * 读取正整数系统属性。 + * + * @param name 属性名 + * @param defaultValue 默认值 + * @return 有效属性值或默认值 + */ + private static int positiveIntProperty(String name, int defaultValue) { + try { + int value = Integer.parseInt(System.getProperty(name, String.valueOf(defaultValue))); + return value > 0 ? value : defaultValue; + } catch (NumberFormatException exception) { + return defaultValue; + } + } + + /** + * 读取正长整数系统属性。 + * + * @param name 属性名 + * @param defaultValue 默认值 + * @return 有效属性值或默认值 + */ + private static long positiveLongProperty(String name, long defaultValue) { + try { + long value = Long.parseLong(System.getProperty(name, String.valueOf(defaultValue))); + return value > 0L ? value : defaultValue; + } catch (NumberFormatException exception) { + return defaultValue; + } + } + + /** + * 按资源类别创建具有独立容量的隔离器。 + * + * @param prefix 系统属性前缀 + * @param globalConcurrency 默认总并发 + * @param targetConcurrency 默认单目标并发 + * @param timeoutMillis 默认等待毫秒数 + * @param trackedTargets 默认目标上限 + * @return 独立隔离器 + */ + private static IoBulkhead lane( + String prefix, + int globalConcurrency, + int targetConcurrency, + long timeoutMillis, + int trackedTargets) { + return new IoBulkhead( + positiveIntProperty( + prefix + ".max-concurrency", + globalConcurrency), + positiveIntProperty( + prefix + ".per-target-max-concurrency", + targetConcurrency), + Duration.ofMillis(positiveLongProperty( + prefix + ".acquire-timeout-ms", + timeoutMillis)), + positiveIntProperty( + prefix + ".max-tracked-targets", + trackedTargets)); + } + + /** + * 一次 I/O 许可,关闭时幂等释放资源。 + */ + public static final class Permit implements AutoCloseable { + + private final IoBulkhead owner; + private final Semaphore targetSemaphore; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建许可。 + * + * @param owner 所属隔离器 + * @param targetSemaphore 目标信号量 + */ + private Permit(IoBulkhead owner, Semaphore targetSemaphore) { + this.owner = owner; + this.targetSemaphore = targetSemaphore; + } + + /** + * 幂等释放许可。 + */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + owner.release(targetSemaphore); + } + } + } + + /** + * I/O 隔离器的不可变运行指标快照。 + * + * @param acquiredCount 已获取许可次数 + * @param rejectedCount 被拒绝次数 + * @param inFlightCount 当前执行中数量 + * @param totalWaitNanos 累计等待纳秒数 + * @param availableGlobalPermits 当前可用全局许可数 + * @param trackedTargetCount 当前独立跟踪目标数 + */ + public record Snapshot( + long acquiredCount, + long rejectedCount, + long inFlightCount, + long totalWaitNanos, + int availableGlobalPermits, + int trackedTargetCount) { + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/IterableUtil.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/IterableUtil.java index 4759226..1c5745d 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/IterableUtil.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/IterableUtil.java @@ -1,5 +1,6 @@ package com.easyagents.flow.core.util; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -61,4 +62,25 @@ public class IterableUtil { throw new IndexOutOfBoundsException("index >= size: " + index); } + + /** + * 单次遍历并物化 Iterable,避免后续按索引从头重复扫描。 + * + * @param iterable 可迭代对象 + * @param 元素类型 + * @return 保持原始迭代顺序的列表 + */ + public static List toList(Iterable iterable) { + if (iterable == null) { + return new ArrayList<>(); + } + if (iterable instanceof Collection) { + return new ArrayList<>((Collection) iterable); + } + List result = new ArrayList<>(); + for (T item : iterable) { + result.add(item); + } + return result; + } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/JsConditionUtil.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/JsConditionUtil.java index 45d1341..0df612c 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/JsConditionUtil.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/JsConditionUtil.java @@ -18,21 +18,84 @@ package com.easyagents.flow.core.util; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.util.graalvm.JsInteropUtils; import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Engine; import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.Source; import org.graalvm.polyglot.Value; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +/** + * 在隔离的 GraalVM JavaScript 上下文中执行工作流条件表达式。 + */ public class JsConditionUtil { - // 使用 Context.Builder 构建上下文,线程安全 - private static final Context.Builder CONTEXT_BUILDER = Context.newBuilder("js") + private static final int SOURCE_CACHE_LIMIT = 2048; + /** + * 单条超长动态表达式不进入共享缓存,限制源码字符串和编译元数据总占用。 + */ + private static final int MAX_CACHEABLE_SOURCE_CHARS = + 16 * 1024; + /** + * Engine 跨 Context 共享编译缓存,Context 仍按每次求值独立创建。 + */ + private static final Engine ENGINE = Engine.newBuilder() .option("engine.WarnInterpreterOnly", "false") + .build(); + private static final Map SOURCE_CACHE = + Collections.synchronizedMap(new LinkedHashMap<>( + SOURCE_CACHE_LIMIT + 1, 0.75F, true) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + return size() > SOURCE_CACHE_LIMIT; + } + }); + + /** + * 工具类禁止实例化。 + */ + private JsConditionUtil() { + } + + /** + * 创建一次隔离的 JavaScript 执行上下文。 + * + * @return 新的执行上下文 + */ + private static Context createContext() { + return Context.newBuilder("js") + .engine(ENGINE) .allowHostAccess(HostAccess.ALL) // 允许访问 Java 对象的方法和字段 .allowHostClassLookup(className -> false) // 禁止动态加载任意 Java 类 - .option("js.ecmascript-version", "2021"); // 使用较新的 ECMAScript 版本 + .option("js.ecmascript-version", "2021") + .build(); + } + + /** + * 获取复用的条件表达式源对象。 + * + * @param code 原始表达式 + * @return 包含结果赋值逻辑的 JavaScript 源 + */ + private static Source source(String code) { + if (code.length() + > MAX_CACHEABLE_SOURCE_CHARS) { + return Source.create( + "js", + "_result.value = " + code); + } + synchronized (SOURCE_CACHE) { + return SOURCE_CACHE.computeIfAbsent( + code, + expression -> Source.create( + "js", + "_result.value = " + expression)); + } + } /** * 执行 JavaScript 表达式并返回 boolean 结果 @@ -43,7 +106,7 @@ public class JsConditionUtil { * @return true 表示满足条件,继续执行;false 表示跳过 */ public static boolean eval(String code, Chain chain, Map initMap) { - try (Context context = CONTEXT_BUILDER.build()) { + try (Context context = createContext()) { Map _result = new HashMap<>(); Value bindings = context.getBindings("js"); @@ -54,9 +117,8 @@ public class JsConditionUtil { }); bindings.putMember("_result", _result); - code = "_result.value = " + code; - context.eval("js", code); + context.eval(source(code)); Object value = _result.get("value"); return toBoolean(value); } catch (Exception e) { @@ -65,8 +127,16 @@ public class JsConditionUtil { } + /** + * 执行 JavaScript 表达式并转换为长整数。 + * + * @param code JS 表达式 + * @param chain Chain 上下文对象 + * @param initMap 初始变量映射 + * @return 转换后的长整数 + */ public static long evalLong(String code, Chain chain, Map initMap) { - try (Context context = CONTEXT_BUILDER.build()) { + try (Context context = createContext()) { Map _result = new HashMap<>(); Value bindings = context.getBindings("js"); @@ -77,9 +147,8 @@ public class JsConditionUtil { }); bindings.putMember("_result", _result); - code = "_result.value = " + code; - context.eval("js", code); + context.eval(source(code)); Object value = _result.get("value"); return toLong(value); } catch (Exception e) { @@ -89,7 +158,10 @@ public class JsConditionUtil { /** - * 将任意对象安全转换为 long 类型 + * 将任意对象安全转换为 long 类型。 + * + * @param value 待转换值 + * @return 长整数结果 */ private static long toLong(Object value) { if (value == null) { @@ -136,13 +208,17 @@ public class JsConditionUtil { } /** - * 收集上下文中的变量 + * 收集上下文中的变量。 + * + * @param chain 当前工作流 + * @param initMap 初始变量 + * @return JavaScript 变量映射 */ private static Map collectContextVariables(Chain chain, Map initMap) { - Map variables = new ConcurrentHashMap<>(); + Map variables = new HashMap<>(); // 添加 Chain Memory 中的变量(去掉前缀) - chain.getState().getMemory().forEach((key, value) -> { + chain.getExecutionState().getMemory().forEach((key, value) -> { int dotIndex = key.indexOf("."); String varName = (dotIndex >= 0) ? key.substring(dotIndex + 1) : key; variables.put(varName, value); @@ -155,7 +231,10 @@ public class JsConditionUtil { } /** - * 将任意对象转换为布尔值 + * 将任意对象转换为布尔值。 + * + * @param value 待转换值 + * @return 布尔结果 */ private static boolean toBoolean(Object value) { if (value == null) { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/OkHttpClientUtil.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/OkHttpClientUtil.java index 3182dbf..38fda73 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/OkHttpClientUtil.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/OkHttpClientUtil.java @@ -16,6 +16,13 @@ package com.easyagents.flow.core.util; import okhttp3.OkHttpClient; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.BufferedSource; +import okio.ForwardingSource; +import okio.Okio; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSocketFactory; @@ -43,6 +50,8 @@ public final class OkHttpClientUtil { private static final Logger LOGGER = Logger.getLogger(OkHttpClientUtil.class.getName()); private static volatile OkHttpClient.Builder customBuilder; + private static volatile OkHttpClient sharedClient; + private static volatile OkHttpClient sharedNoRetryClient; private static final Object LOCK = new Object(); // Prevent instantiation @@ -58,7 +67,11 @@ public final class OkHttpClientUtil { if (builder == null) { throw new IllegalArgumentException("Builder must not be null"); } - customBuilder = builder; + synchronized (LOCK) { + customBuilder = builder; + sharedClient = null; + sharedNoRetryClient = null; + } } /** @@ -70,31 +83,73 @@ public final class OkHttpClientUtil { *

*/ public static OkHttpClient buildDefaultClient() { - OkHttpClient.Builder builder = customBuilder; - if (builder != null) { - return builder.build(); + OkHttpClient client = sharedClient; + if (client != null) { + return client; } synchronized (LOCK) { - // Double-check in case another thread set it while waiting - builder = customBuilder; - if (builder != null) { - return builder.build(); + client = sharedClient; + if (client != null) { + return client; } - builder = new OkHttpClient.Builder() - .connectTimeout(1, TimeUnit.MINUTES) - .readTimeout(5, TimeUnit.MINUTES); + OkHttpClient.Builder builder = customBuilder; + if (builder == null) { + builder = new OkHttpClient.Builder() + .connectTimeout(1, TimeUnit.MINUTES) + .readTimeout(5, TimeUnit.MINUTES); - // Optional insecure mode (for development/testing only) - if (isInsecureModeEnabled()) { - LOGGER.warning("OkHttpClient is running in INSECURE mode (trust-all SSL). " + - "This is dangerous and should not be used in production."); - enableInsecureSsl(builder); + // Optional insecure mode (for development/testing only) + if (isInsecureModeEnabled()) { + LOGGER.warning("OkHttpClient is running in INSECURE mode (trust-all SSL). " + + "This is dangerous and should not be used in production."); + enableInsecureSsl(builder); + } + + configureProxy(builder); } + configureIoBulkhead(builder); + sharedClient = builder.build(); + return sharedClient; + } + } - configureProxy(builder); - return builder.build(); + /** + * 返回关闭 OkHttp 隐式连接重试的共享客户端。 + * + *

该客户端与默认客户端复用连接池和调度器,用于可能产生业务副作用的请求。

+ * + * @return 禁用连接失败自动重试的共享客户端 + */ + public static OkHttpClient buildNoRetryClient() { + OkHttpClient client = sharedNoRetryClient; + if (client != null) { + return client; + } + synchronized (LOCK) { + client = sharedNoRetryClient; + if (client == null) { + client = buildDefaultClient() + .newBuilder() + .retryOnConnectionFailure(false) + .build(); + sharedNoRetryClient = client; + } + return client; + } + } + + /** + * 为共享客户端安装一次工作流 I/O 隔离拦截器。 + * + * @param builder HTTP 客户端构建器 + */ + private static void configureIoBulkhead(OkHttpClient.Builder builder) { + boolean configured = builder.interceptors().stream() + .anyMatch(interceptor -> interceptor instanceof IoBulkheadInterceptor); + if (!configured) { + builder.addInterceptor(new IoBulkheadInterceptor()); } } @@ -162,4 +217,99 @@ public final class OkHttpClientUtil { } return port; } -} \ No newline at end of file + + /** + * 在 HTTP 响应体关闭前持有 I/O 许可的拦截器。 + */ + private static final class IoBulkheadInterceptor implements Interceptor { + + /** + * 对单个 HTTP 目标施加并发隔离。 + * + * @param chain OkHttp 拦截链 + * @return HTTP 响应 + * @throws java.io.IOException 网络请求失败时抛出 + */ + @Override + public Response intercept(Chain chain) throws java.io.IOException { + IoBulkhead.Permit permit = IoBulkhead.shared() + .acquire(IoBulkhead.targetForUrl(chain.request().url().toString())); + boolean transferred = false; + try { + Response response = chain.proceed(chain.request()); + ResponseBody body = response.body(); + if (body == null) { + return response; + } + Response wrapped = response.newBuilder() + .body(new PermitReleasingResponseBody(body, permit)) + .build(); + transferred = true; + return wrapped; + } finally { + if (!transferred) { + permit.close(); + } + } + } + } + + /** + * 在响应体关闭时释放 I/O 许可的响应体包装器。 + */ + private static final class PermitReleasingResponseBody extends ResponseBody { + + private final ResponseBody delegate; + private final BufferedSource source; + + /** + * 创建响应体包装器。 + * + * @param delegate 原始响应体 + * @param permit 待释放的 I/O 许可 + */ + private PermitReleasingResponseBody(ResponseBody delegate, IoBulkhead.Permit permit) { + this.delegate = delegate; + this.source = Okio.buffer(new ForwardingSource(delegate.source()) { + @Override + public void close() throws java.io.IOException { + try { + super.close(); + } finally { + permit.close(); + } + } + }); + } + + /** + * 获取响应媒体类型。 + * + * @return 响应媒体类型 + */ + @Override + public MediaType contentType() { + return delegate.contentType(); + } + + /** + * 获取响应声明长度。 + * + * @return 响应声明长度 + */ + @Override + public long contentLength() { + return delegate.contentLength(); + } + + /** + * 获取带许可释放逻辑的响应源。 + * + * @return 响应源 + */ + @Override + public BufferedSource source() { + return source; + } + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/TextTemplate.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/TextTemplate.java index 909b84f..eb0a3c7 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/TextTemplate.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/util/TextTemplate.java @@ -46,12 +46,16 @@ public class TextTemplate { /** * 模板缓存(按原始模板字符串) */ - private static final Map TEMPLATE_CACHE = new ConcurrentHashMap<>(); + private static final int TEMPLATE_CACHE_LIMIT = 4096; + private static final int JSONPATH_CACHE_LIMIT = 2048; + private static final Map TEMPLATE_CACHE = + Collections.synchronizedMap(new BoundedLruMap<>(TEMPLATE_CACHE_LIMIT)); /** * JSONPath 编译缓存,避免重复编译 */ - private static final Map JSONPATH_CACHE = new ConcurrentHashMap<>(); + private static final Map JSONPATH_CACHE = + Collections.synchronizedMap(new BoundedLruMap<>(JSONPATH_CACHE_LIMIT)); /** * 原始模板字符串 @@ -73,7 +77,9 @@ public class TextTemplate { */ public static TextTemplate of(String template) { String finalTemplate = template != null ? template : ""; - return MapUtil.computeIfAbsent(TEMPLATE_CACHE, finalTemplate, k -> new TextTemplate(finalTemplate)); + synchronized (TEMPLATE_CACHE) { + return TEMPLATE_CACHE.computeIfAbsent(finalTemplate, TextTemplate::new); + } } /** @@ -86,13 +92,42 @@ public class TextTemplate { public String formatToString(List> rootMaps) { - Map rootMap = new HashMap<>(); - for (Map m : rootMaps) { - if (m != null) { - rootMap.putAll(m); - } + return formatToString(rootMaps, false); + } + + /** + * 使用分层上下文格式化模板,避免合并复制完整工作流状态。 + *

+ * 后面的上下文层优先级更高,与原有 Map 合并顺序一致。 + * + * @param rootMaps 分层模板上下文 + * @param escapeForJsonOutput 是否对结果进行 JSON 字符串转义 + * @return 格式化结果 + */ + public String formatToString(List> rootMaps, boolean escapeForJsonOutput) { + if (tokens.isEmpty()) { + return originalTemplate; } - return formatToString(rootMap, false); + List> contexts = rootMaps == null ? Collections.emptyList() : rootMaps; + Map layeredContext = new LayeredContextMap(contexts); + StringBuilder result = new StringBuilder(originalTemplate.length() + 64); + for (TemplateToken token : tokens) { + if (token.isStatic) { + result.append(token.content); + continue; + } + EvaluationResult evaluationResult = evaluate( + token.parseResult, layeredContext, escapeForJsonOutput); + if (!token.explicitEmptyFallback && !evaluationResult.resolved) { + throw new IllegalArgumentException(String.format( + "Missing value for expression: \"%s\"%nTemplate: %s%nProvided context layers: %d", + token.rawExpression, + originalTemplate, + contexts.size())); + } + result.append(evaluationResult.value); + } + return result.toString(); } /** @@ -236,7 +271,10 @@ public class TextTemplate { } String fullPath = path.startsWith("$") ? path : "$." + path; - JSONPath compiled = MapUtil.computeIfAbsent(JSONPATH_CACHE, fullPath, JSONPath::compile); + JSONPath compiled; + synchronized (JSONPATH_CACHE) { + compiled = JSONPATH_CACHE.computeIfAbsent(fullPath, JSONPath::compile); + } Object value = compiled.eval(root); if (escapeForJsonOutput && value instanceof String) { return escapeJsonString((String) value); @@ -384,4 +422,159 @@ public class TextTemplate { return new EvaluationResult(false, ""); } } + + /** + * 保持 putAll 覆盖语义、但不复制底层数据的只读分层 Map。 + */ + private static final class LayeredContextMap extends AbstractMap { + + private final List> layers; + + private LayeredContextMap(List> layers) { + this.layers = layers; + } + + @Override + public Object get(Object key) { + for (int index = layers.size() - 1; index >= 0; index--) { + Map layer = layers.get(index); + if (layer != null && layer.containsKey(key)) { + return layer.get(key); + } + } + return null; + } + + @Override + public boolean containsKey(Object key) { + for (int index = layers.size() - 1; index >= 0; index--) { + Map layer = layers.get(index); + if (layer != null && layer.containsKey(key)) { + return true; + } + } + return false; + } + + @Override + public boolean isEmpty() { + for (Map layer : layers) { + if (layer != null && !layer.isEmpty()) { + return false; + } + } + return true; + } + + @Override + public int size() { + return keySet().size(); + } + + @Override + public Set keySet() { + Set keys = new LinkedHashSet<>(); + for (Map layer : layers) { + if (layer != null) { + keys.addAll(layer.keySet()); + } + } + return Collections.unmodifiableSet(keys); + } + + @Override + public Set> entrySet() { + Set keys = keySet(); + return new AbstractSet<>() { + @Override + public Iterator> iterator() { + Iterator iterator = + keys.iterator(); + return new Iterator<>() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Entry next() { + String key = iterator.next(); + return lazyEntry(key); + } + }; + } + + @Override + public int size() { + return keys.size(); + } + }; + } + + /** + * 创建仅在读取值时访问底层上下文的不可变条目。 + * + * @param key 上下文键 + * @return 惰性条目 + */ + private Entry lazyEntry( + String key) { + return new Entry<>() { + @Override + public String getKey() { + return key; + } + + @Override + public Object getValue() { + return LayeredContextMap.this + .get(key); + } + + @Override + public Object setValue(Object value) { + throw new UnsupportedOperationException( + "read-only template context"); + } + + @Override + public boolean equals(Object value) { + return value instanceof Entry entry + && Objects.equals( + key, entry.getKey()) + && Objects.equals( + getValue(), + entry.getValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(key) + ^ Objects.hashCode( + getValue()); + } + }; + } + } + + /** + * 固定容量的最近最少使用缓存。 + * + * @param 键类型 + * @param 值类型 + */ + private static final class BoundedLruMap extends LinkedHashMap { + + private final int limit; + + private BoundedLruMap(int limit) { + super(16, 0.75F, true); + this.limit = limit; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > limit; + } + } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/node/HttpNodePerformanceSafetyTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/node/HttpNodePerformanceSafetyTest.java new file mode 100644 index 0000000..b5d6e13 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/node/HttpNodePerformanceSafetyTest.java @@ -0,0 +1,126 @@ +package com.easyagents.flow.core.node; + +import com.easyagents.flow.core.chain.Chain; +import okhttp3.MediaType; +import okhttp3.ResponseBody; +import okio.Buffer; +import okio.BufferedSource; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +/** + * {@link HttpNode} 性能保护与重试安全回归测试。 + */ +public class HttpNodePerformanceSafetyTest { + + /** + * 验证非幂等请求不会执行节点内部隐式自动重试。 + */ + @Test + public void shouldNotAutomaticallyRetryNonIdempotentMethod() { + FailingHttpNode node = new FailingHttpNode(); + node.setMethod("POST"); + + try { + node.execute(null); + Assert.fail("request should fail"); + } catch (RuntimeException expected) { + Assert.assertEquals(1, node.attempts); + } + } + + /** + * 验证仅读取类 HTTP 方法允许节点内部自动重试。 + */ + @Test + public void shouldOnlyRetrySafeReadMethods() { + FailingHttpNode node = new FailingHttpNode(); + + Assert.assertTrue(node.allowsAutomaticRetry("GET")); + Assert.assertTrue(node.allowsAutomaticRetry("HEAD")); + Assert.assertTrue(node.allowsAutomaticRetry("OPTIONS")); + Assert.assertFalse(node.allowsAutomaticRetry("POST")); + Assert.assertFalse(node.allowsAutomaticRetry("PUT")); + Assert.assertFalse(node.allowsAutomaticRetry("PATCH")); + Assert.assertFalse(node.allowsAutomaticRetry("DELETE")); + } + + /** + * 验证响应头未声明长度时仍按实际读取量拒绝超限文本。 + * + * @throws Exception 响应读取失败时抛出 + */ + @Test + public void shouldEnforceActualTextResponseSize() throws Exception { + FailingHttpNode node = new FailingHttpNode(); + ResponseBody body = new ResponseBody() { + @Override + public MediaType contentType() { + return MediaType.parse("text/plain; charset=utf-8"); + } + + @Override + public long contentLength() { + return -1L; + } + + @Override + public BufferedSource source() { + return new Buffer().writeUtf8("12345"); + } + }; + + try { + node.read(body, 4L); + Assert.fail("actual response bytes should be limited"); + } catch (IOException exception) { + Assert.assertTrue(exception.getMessage().contains("4 bytes")); + } + } + + /** + * 始终返回 I/O 异常的测试节点。 + */ + private static final class FailingHttpNode extends HttpNode { + + private int attempts; + + /** + * 模拟单次 HTTP 调用失败。 + * + * @param chain 工作流链 + * @return 不会正常返回 + * @throws IOException 固定抛出模拟异常 + */ + @Override + public Map doExecute(Chain chain) throws IOException { + attempts++; + throw new IOException("simulated failure"); + } + + /** + * 暴露自动重试判断供测试使用。 + * + * @param method HTTP 方法 + * @return 是否允许自动重试 + */ + private boolean allowsAutomaticRetry(String method) { + return supportsAutomaticRetry(method); + } + + /** + * 暴露文本响应读取供测试使用。 + * + * @param body 响应体 + * @param maxBytes 最大允许字节数 + * @return 响应文本 + * @throws IOException 读取失败或超限时抛出 + */ + private String read(ResponseBody body, long maxBytes) throws IOException { + return readTextBody(body, maxBytes); + } + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainDefinitionIndexTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainDefinitionIndexTest.java new file mode 100644 index 0000000..2ff3af0 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainDefinitionIndexTest.java @@ -0,0 +1,128 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.Edge; +import com.easyagents.flow.core.node.EndNode; +import com.easyagents.flow.core.node.StartNode; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Collections; + +/** + * 验证工作流定义图索引的查询结果和变更失效行为。 + */ +public class ChainDefinitionIndexTest { + + /** + * 验证节点、边、邻接表和开始节点保持原有查询语义。 + */ + @Test + public void shouldQueryDefinitionThroughGraphIndex() { + ChainDefinition definition = new ChainDefinition(); + StartNode startNode = node(new StartNode(), "start"); + EndNode endNode = node(new EndNode(), "end"); + Edge edge = edge("start-to-end", "start", "end"); + + definition.addNode(startNode); + definition.addNode(endNode); + definition.addEdge(edge); + + Assert.assertSame(startNode, definition.getNodeById("start")); + Assert.assertSame(edge, definition.getEdgeById("start-to-end")); + Assert.assertEquals(Collections.singletonList(edge), definition.getOutwardEdge("start")); + Assert.assertEquals(Collections.singletonList(edge), definition.getInwardEdge("end")); + Assert.assertEquals(Collections.singletonList(startNode), definition.getStartNodes()); + } + + /** + * 验证定义修改后会重建图索引,不返回过期结果。 + */ + @Test + public void shouldInvalidateGraphIndexAfterDefinitionChanges() { + ChainDefinition definition = new ChainDefinition(); + StartNode startNode = node(new StartNode(), "start"); + EndNode firstEnd = node(new EndNode(), "end-1"); + definition.addNode(startNode); + definition.addNode(firstEnd); + definition.addEdge(edge("edge-1", "start", "end-1")); + + Assert.assertEquals(1, definition.getOutwardEdge("start").size()); + + EndNode secondEnd = node(new EndNode(), "end-2"); + definition.addNode(secondEnd); + definition.addEdge(edge("edge-2", "start", "end-2")); + + Assert.assertSame(secondEnd, definition.getNodeById("end-2")); + Assert.assertEquals(2, definition.getOutwardEdge("start").size()); + Assert.assertEquals(Collections.singletonList(startNode), definition.getStartNodes()); + } + + /** + * 验证包含节点、边和条件的定义可作为实例快照完整序列化。 + * + * @throws Exception 序列化或反序列化失败 + */ + @Test + public void shouldRoundTripDefinitionSnapshot() throws Exception { + ChainDefinition definition = new ChainDefinition(); + StartNode startNode = node(new StartNode(), "start"); + EndNode endNode = node(new EndNode(), "end"); + Edge edge = edge("start-to-end", "start", "end"); + edge.setCondition((chain, currentEdge, result) -> true); + definition.addNode(startNode); + definition.addNode(endNode); + definition.addEdge(edge); + + byte[] bytes; + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + ObjectOutputStream objectOutput = new ObjectOutputStream(output)) { + objectOutput.writeObject(definition); + bytes = output.toByteArray(); + } + + ChainDefinition restored; + try (ObjectInputStream objectInput = + new ObjectInputStream(new ByteArrayInputStream(bytes))) { + restored = (ChainDefinition) objectInput.readObject(); + } + + Assert.assertEquals(2, restored.getNodes().size()); + Assert.assertEquals(1, restored.getEdges().size()); + Assert.assertEquals("end", restored.getOutwardEdge("start").get(0).getTarget()); + Assert.assertNotNull(restored.getOutwardEdge("start").get(0).getCondition()); + } + + /** + * 为测试节点设置 ID。 + * + * @param node 节点 + * @param id 节点 ID + * @param 节点类型 + * @return 设置完成的节点 + */ + private T node(T node, String id) { + node.setId(id); + return node; + } + + /** + * 创建测试边。 + * + * @param id 边 ID + * @param source 来源节点 ID + * @param target 目标节点 ID + * @return 测试边 + */ + private Edge edge(String id, String source, String target) { + Edge edge = new Edge(); + edge.setId(id); + edge.setSource(source); + edge.setTarget(target); + return edge; + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java index 482736c..f1547eb 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java @@ -15,20 +15,31 @@ */ package com.easyagents.flow.core.test; +import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainStatus; import com.easyagents.flow.core.chain.Edge; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.repository.ChainDefinitionSnapshotRepository; +import com.easyagents.flow.core.chain.repository.ChainStateField; +import com.easyagents.flow.core.chain.repository.ChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryChainDefinitionSnapshotRepository; import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; import com.easyagents.flow.core.chain.runtime.ChainExecutor; import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; import com.easyagents.flow.core.chain.runtime.TriggerScheduler; import com.easyagents.flow.core.node.EndNode; +import com.easyagents.flow.core.node.BaseNode; import com.easyagents.flow.core.node.StartNode; import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -37,6 +48,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * {@link ChainExecutor} 并发同步执行测试。 @@ -85,6 +97,298 @@ public class ChainExecutorConcurrencyTest { } } + /** + * 验证一个工作流实例的多个节点触发会复用启动时的定义快照。 + */ + @Test + public void shouldReuseDefinitionSnapshotAcrossNodeTriggers() { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newFixedThreadPool(2); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + ChainDefinition definition = createDefinition(); + AtomicInteger definitionLoadCount = new AtomicInteger(); + ChainExecutor chainExecutor = new ChainExecutor( + id -> { + definitionLoadCount.incrementAndGet(); + return definition; + }, + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository(), + triggerScheduler); + + try { + Map result = chainExecutor.execute( + definition.getId(), Collections.emptyMap(), 10, TimeUnit.SECONDS); + + Assert.assertNotNull(result); + Assert.assertEquals(1, definitionLoadCount.get()); + } finally { + triggerScheduler.shutdown(); + } + } + + /** + * 验证取消期间已经完成的节点 I/O 不会继续推进下游节点。 + * + * @throws Exception 等待异步节点进入或退出失败时抛出 + */ + @Test + public void shouldNotAdvanceAfterCancellationDuringNodeIo() throws Exception { + ScheduledExecutorService schedulerPool = Executors.newScheduledThreadPool(2); + ExecutorService workerPool = Executors.newFixedThreadPool(2); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + InMemoryChainStateRepository chainStateRepository = new InMemoryChainStateRepository(); + CountDownLatch ioStarted = new CountDownLatch(1); + CountDownLatch allowIoCompletion = new CountDownLatch(1); + AtomicInteger downstreamExecutions = new AtomicInteger(); + ChainDefinition definition = createCancellationDefinition( + ioStarted, allowIoCompletion, downstreamExecutions); + ChainExecutor chainExecutor = new ChainExecutor( + id -> definition, + chainStateRepository, + new InMemoryNodeStateRepository(), + triggerScheduler); + + try { + String instanceId = chainExecutor.executeAsync( + definition.getId(), Collections.emptyMap()); + Assert.assertTrue(ioStarted.await(2, TimeUnit.SECONDS)); + Assert.assertTrue(chainExecutor.cancel(instanceId, "test cancellation")); + allowIoCompletion.countDown(); + + Thread.sleep(200L); + Assert.assertEquals(0, downstreamExecutions.get()); + Assert.assertEquals( + ChainStatus.CANCELLED, + chainStateRepository.load(instanceId).getStatus()); + } finally { + allowIoCompletion.countDown(); + triggerScheduler.shutdown(); + } + } + + /** + * 验证同一节点执行期间重复读取状态会复用执行快照。 + */ + @Test + public void shouldKeepPublicStateReadsFreshDuringNodeExecution() { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newSingleThreadExecutor(); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + CountingChainStateRepository stateRepository = new CountingChainStateRepository(); + AtomicInteger nodeStateLoads = new AtomicInteger(-1); + ChainDefinition definition = createStateReadDefinition( + stateRepository.loadCount, + nodeStateLoads, + false); + ChainExecutor chainExecutor = new ChainExecutor( + id -> definition, + stateRepository, + new InMemoryNodeStateRepository(), + triggerScheduler); + + try { + Map result = chainExecutor.execute( + definition.getId(), Collections.emptyMap(), 10, TimeUnit.SECONDS); + + Assert.assertNotNull(result); + Assert.assertEquals(10, nodeStateLoads.get()); + } finally { + triggerScheduler.shutdown(); + } + } + + /** + * 验证节点内部执行视图重复读取不会访问状态仓储。 + */ + @Test + public void shouldReusePointInTimeStateDuringNodeExecution() { + ScheduledExecutorService schedulerPool = + Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = + Executors.newSingleThreadExecutor(); + TriggerScheduler triggerScheduler = + new TriggerScheduler( + new InMemoryTriggerStore(), + schedulerPool, + workerPool, + 1000L); + CountingChainStateRepository stateRepository = + new CountingChainStateRepository(); + AtomicInteger nodeStateLoads = + new AtomicInteger(-1); + ChainDefinition definition = + createStateReadDefinition( + stateRepository.loadCount, + nodeStateLoads, + true); + ChainExecutor chainExecutor = + new ChainExecutor( + id -> definition, + stateRepository, + new InMemoryNodeStateRepository(), + triggerScheduler); + + try { + Map result = + chainExecutor.execute( + definition.getId(), + Collections.emptyMap(), + 10, + TimeUnit.SECONDS); + + Assert.assertNotNull(result); + Assert.assertEquals( + 0, nodeStateLoads.get()); + } finally { + triggerScheduler.shutdown(); + } + } + + /** + * 验证升级前 parent-linked 状态首次解析顶级审计 ID 后会持久复用。 + */ + @Test + public void shouldBackfillLegacyAuditInstanceIdOnce() { + CountingChainStateRepository repository = + new CountingChainStateRepository(); + ChainState root = + repository.create("audit-root"); + ChainState child = + repository.create("audit-child"); + child.setParentInstanceId( + root.getInstanceId()); + child.setAuditInstanceId(null); + ChainDefinition definition = + new ChainDefinition(); + definition.setId("audit-definition"); + Chain chain = + new Chain( + definition, + child.getInstanceId()); + chain.setChainStateRepository( + repository); + + Assert.assertEquals( + root.getInstanceId(), + chain.getAuditInstanceId()); + int loadsAfterBackfill = + repository.loadCount.get(); + Assert.assertEquals( + root.getInstanceId(), + chain.getAuditInstanceId()); + + Assert.assertEquals( + loadsAfterBackfill + 1, + repository.loadCount.get()); + Assert.assertEquals( + root.getInstanceId(), + repository.load( + child.getInstanceId()) + .getAuditInstanceId()); + } + + /** + * 验证实例状态初始化失败时会清理进程缓存和持久定义快照。 + */ + @Test + public void shouldCleanupDefinitionSnapshotWhenInitializationFails() throws Exception { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newSingleThreadExecutor(); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + TrackingSnapshotRepository snapshotRepository = new TrackingSnapshotRepository(); + ChainStateRepository failingStateRepository = new ChainStateRepository() { + @Override + public ChainState load(String instanceId) { + return null; + } + + @Override + public ChainState create(String instanceId) { + throw new IllegalStateException("initialization failed"); + } + + @Override + public boolean tryUpdate( + ChainState newState, EnumSet fields) { + return false; + } + }; + ChainExecutor executor = new ChainExecutor( + id -> createDefinition(), + failingStateRepository, + new InMemoryNodeStateRepository(), + null, + snapshotRepository, + triggerScheduler, + null); + + try { + executor.executeAsync("concurrent-sync-test", Collections.emptyMap()); + Assert.fail("initialization failure must be propagated"); + } catch (IllegalStateException expected) { + Assert.assertEquals("initialization failed", expected.getMessage()); + } finally { + triggerScheduler.shutdown(); + } + + Assert.assertTrue(snapshotRepository.snapshots.isEmpty()); + Assert.assertTrue(activeDefinitions(executor).isEmpty()); + } + + /** + * 验证进程内定义热点缓存有固定上限,持久快照仍保留恢复能力。 + */ + @Test + public void shouldBoundActiveDefinitionCache() throws Exception { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newSingleThreadExecutor(); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + InMemoryChainDefinitionSnapshotRepository snapshotRepository = + new InMemoryChainDefinitionSnapshotRepository(); + ChainExecutor executor = new ChainExecutor( + id -> createDefinition(), + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository(), + null, + snapshotRepository, + triggerScheduler, + null); + java.lang.reflect.Method createChain = ChainExecutor.class.getDeclaredMethod( + "createChain", ChainDefinition.class); + createChain.setAccessible(true); + + try { + for (int index = 0; index < 1100; index++) { + createChain.invoke(executor, createDefinition()); + } + } finally { + triggerScheduler.shutdown(); + } + + Assert.assertEquals(1024, activeDefinitions(executor).size()); + } + + /** + * 读取执行器内部定义缓存,供资源上限回归验证。 + * + * @param executor 工作流执行器 + * @return 当前定义缓存 + * @throws Exception 反射访问失败时抛出 + */ + @SuppressWarnings("unchecked") + private Map activeDefinitions( + ChainExecutor executor) throws Exception { + Field field = ChainExecutor.class.getDeclaredField("activeDefinitions"); + field.setAccessible(true); + return (Map) field.get(executor); + } + /** * 创建仅包含开始和结束节点的测试工作流。 * @@ -109,4 +413,155 @@ public class ChainExecutorConcurrencyTest { definition.addEdge(edge); return definition; } + + /** + * 创建用于取消传播验证的工作流。 + * + * @param ioStarted I/O 节点进入信号 + * @param allowIoCompletion I/O 节点退出信号 + * @param downstreamExecutions 下游执行计数 + * @return 测试工作流定义 + */ + private ChainDefinition createCancellationDefinition(CountDownLatch ioStarted, + CountDownLatch allowIoCompletion, + AtomicInteger downstreamExecutions) { + ChainDefinition definition = new ChainDefinition(); + definition.setId("cancellation-test"); + + StartNode start = new StartNode(); + start.setId("start"); + BaseNode blocking = new BaseNode() { + @Override + public Map execute(Chain chain) { + ioStarted.countDown(); + try { + if (!allowIoCompletion.await(2, TimeUnit.SECONDS)) { + throw new IllegalStateException("test I/O release timed out"); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("test I/O interrupted", error); + } + return Collections.singletonMap("value", "completed"); + } + }; + blocking.setId("blocking"); + BaseNode downstream = new BaseNode() { + @Override + public Map execute(Chain chain) { + downstreamExecutions.incrementAndGet(); + return Collections.emptyMap(); + } + }; + downstream.setId("downstream"); + EndNode end = new EndNode(); + end.setId("end"); + + definition.addNode(start); + definition.addNode(blocking); + definition.addNode(downstream); + definition.addNode(end); + definition.addEdge(edge("start-blocking", "start", "blocking")); + definition.addEdge(edge("blocking-downstream", "blocking", "downstream")); + definition.addEdge(edge("downstream-end", "downstream", "end")); + return definition; + } + + /** + * 创建包含重复状态读取节点的测试工作流。 + * + * @param repositoryLoadCount 仓储读取计数 + * @param nodeStateLoads 节点执行期间新增的读取次数 + * @param useExecutionView 是否读取节点 point-in-time 执行视图 + * @return 测试工作流定义 + */ + private ChainDefinition createStateReadDefinition(AtomicInteger repositoryLoadCount, + AtomicInteger nodeStateLoads, + boolean useExecutionView) { + ChainDefinition definition = new ChainDefinition(); + definition.setId("state-snapshot-test"); + + StartNode start = new StartNode(); + start.setId("start"); + BaseNode repeatedReader = new BaseNode() { + @Override + public Map execute(Chain chain) { + int before = repositoryLoadCount.get(); + for (int index = 0; index < 10; index++) { + Assert.assertNotNull( + useExecutionView + ? chain.getExecutionState() + : chain.getState()); + } + nodeStateLoads.set(repositoryLoadCount.get() - before); + return Collections.emptyMap(); + } + }; + repeatedReader.setId("reader"); + EndNode end = new EndNode(); + end.setId("end"); + + definition.addNode(start); + definition.addNode(repeatedReader); + definition.addNode(end); + definition.addEdge(edge("start-reader", "start", "reader")); + definition.addEdge(edge("reader-end", "reader", "end")); + return definition; + } + + /** + * 创建测试边。 + * + * @param id 边 ID + * @param source 源节点 ID + * @param target 目标节点 ID + * @return 测试边 + */ + private Edge edge(String id, String source, String target) { + Edge edge = new Edge(); + edge.setId(id); + edge.setSource(source); + edge.setTarget(target); + return edge; + } + + /** + * 记录定义快照生命周期的测试仓储。 + */ + private static final class TrackingSnapshotRepository + implements ChainDefinitionSnapshotRepository { + + private final Map snapshots = + new LinkedHashMap<>(); + + @Override + public void save(String instanceId, ChainDefinition definition) { + snapshots.put(instanceId, definition); + } + + @Override + public ChainDefinition load(String instanceId) { + return snapshots.get(instanceId); + } + + @Override + public void remove(String instanceId) { + snapshots.remove(instanceId); + } + } + + /** + * 记录状态读取次数的测试仓储。 + */ + private static final class CountingChainStateRepository + extends InMemoryChainStateRepository { + + private final AtomicInteger loadCount = new AtomicInteger(); + + @Override + public ChainState load(String instanceId) { + loadCount.incrementAndGet(); + return super.load(instanceId); + } + } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainRecoverableStartTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainRecoverableStartTest.java new file mode 100644 index 0000000..08c0138 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainRecoverableStartTest.java @@ -0,0 +1,313 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.EventManager; +import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.NodeStatus; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.node.StartNode; +import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; +import com.easyagents.flow.core.chain.runtime.Trigger; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * 验证入口触发器先行持久化和崩溃重放协议。 + */ +public class ChainRecoverableStartTest { + + /** + * 验证 RUNNING 状态提交前已经存在入口意图,且 RUNNING 节点允许重放。 + */ + @Test + public void shouldPersistIntentBeforeRunningAndReplayInterruptedStart() { + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + ObservingTriggerStore triggerStore = + new ObservingTriggerStore(stateRepository, "recoverable-1"); + ScheduledExecutorService schedulerExecutor = + Executors.newSingleThreadScheduledExecutor(); + java.util.concurrent.ExecutorService worker = + Executors.newSingleThreadExecutor(); + TriggerScheduler scheduler = new TriggerScheduler( + triggerStore, schedulerExecutor, worker, 60_000L); + try { + Chain chain = chain( + stateRepository, triggerStore, scheduler); + chain.start(Collections.singletonMap("value", "v")); + + Assert.assertEquals( + ChainStatus.READY, + triggerStore.statusObservedAtFirstSave); + ChainState running = stateRepository.load("recoverable-1"); + Assert.assertEquals(ChainStatus.RUNNING, running.getStatus()); + Trigger pending = triggerStore.find( + triggerStore.lastTriggerId); + Assert.assertNotNull(pending); + + NodeState startState = chain.getNodeState("start"); + startState.setStatus(NodeStatus.RUNNING); + startState.getExecuteCount().incrementAndGet(); + chain.start(Collections.emptyMap()); + + Assert.assertNotNull( + "interrupted RUNNING start must remain replayable", + triggerStore.find(triggerStore.lastTriggerId)); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证入口意图保存后、RUNNING 提交前崩溃时,真实调度消费可恢复变量并继续执行。 + * + * @throws Exception 等待异步调度被中断 + */ + @Test + public void shouldRecoverReadyStartThroughRealScheduler() + throws Exception { + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + FailAfterSaveTriggerStore triggerStore = + new FailAfterSaveTriggerStore(); + ScheduledExecutorService schedulerExecutor = + Executors.newSingleThreadScheduledExecutor(); + java.util.concurrent.ExecutorService worker = + Executors.newSingleThreadExecutor(); + TriggerScheduler scheduler = new TriggerScheduler( + triggerStore, schedulerExecutor, worker, 1_000L); + try { + Chain chain = chain( + stateRepository, triggerStore, scheduler, + "recoverable-crash"); + StartNode startNode = (StartNode) chain.getDefinition() + .getNodeById("start"); + scheduler.registerConsumer( + (trigger, ignored) -> + chain.executeNode(startNode, trigger)); + + try { + chain.start(Collections.singletonMap( + "value", "persisted-input")); + Assert.fail("simulated crash must interrupt start"); + } catch (SimulatedCrashException expected) { + // 触发器已经保存,异常模拟进程在 RUNNING 提交前退出。 + } + Assert.assertEquals( + ChainStatus.READY, + stateRepository.load( + "recoverable-crash").getStatus()); + + long deadline = System.nanoTime() + + TimeUnit.SECONDS.toNanos(5L); + while (System.nanoTime() < deadline) { + ChainState recovered = stateRepository.load( + "recoverable-crash"); + NodeState nodeState = chain.getNodeState("start"); + if (recovered.getStatus() != ChainStatus.READY + && nodeState.getStatus() + == NodeStatus.SUCCEEDED) { + break; + } + Thread.sleep(25L); + } + + ChainState recovered = stateRepository.load( + "recoverable-crash"); + Assert.assertNotEquals( + ChainStatus.READY, recovered.getStatus()); + Assert.assertEquals( + "persisted-input", + recovered.getMemory().get("value")); + Assert.assertEquals( + NodeStatus.SUCCEEDED, + chain.getNodeState("start").getStatus()); + Assert.assertNull(triggerStore.find( + triggerStore.lastTriggerId)); + } finally { + scheduler.shutdown(); + } + } + + /** + * 创建仅含入口节点的测试链。 + * + * @param stateRepository 状态仓储 + * @param triggerStore 触发器仓储 + * @param scheduler 调度器 + * @return 已配置链 + */ + private Chain chain( + InMemoryChainStateRepository stateRepository, + InMemoryTriggerStore triggerStore, + TriggerScheduler scheduler) { + ChainDefinition definition = new ChainDefinition(); + definition.setId("recoverable-definition"); + StartNode start = new StartNode(); + start.setId("start"); + definition.addNode(start); + definition.setEdges(Collections.emptyList()); + + Chain chain = new Chain(definition, "recoverable-1"); + chain.setChainStateRepository(stateRepository); + chain.setNodeStateRepository( + new InMemoryNodeStateRepository()); + chain.setTriggerScheduler(scheduler); + chain.setEventManager(new EventManager()); + return chain; + } + + /** + * 创建指定实例 ID 的仅入口测试链。 + * + * @param stateRepository 状态仓储 + * @param triggerStore 触发器仓储 + * @param scheduler 调度器 + * @param instanceId 实例 ID + * @return 已配置链 + */ + private Chain chain( + InMemoryChainStateRepository stateRepository, + InMemoryTriggerStore triggerStore, + TriggerScheduler scheduler, + String instanceId) { + ChainDefinition definition = new ChainDefinition(); + definition.setId("recoverable-definition"); + StartNode start = new StartNode(); + start.setId("start"); + definition.addNode(start); + definition.setEdges(Collections.emptyList()); + + Chain chain = new Chain(definition, instanceId); + chain.setChainStateRepository(stateRepository); + chain.setNodeStateRepository( + new InMemoryNodeStateRepository()); + chain.setTriggerScheduler(scheduler); + chain.setEventManager(new EventManager()); + return chain; + } + + /** + * 记录首次触发器保存时的实例状态。 + */ + private static final class ObservingTriggerStore + extends InMemoryTriggerStore { + + private final InMemoryChainStateRepository stateRepository; + private final String instanceId; + private ChainStatus statusObservedAtFirstSave; + private String lastTriggerId; + + /** + * 创建观察仓储。 + * + * @param stateRepository 状态仓储 + * @param instanceId 实例 ID + */ + private ObservingTriggerStore( + InMemoryChainStateRepository stateRepository, + String instanceId) { + this.stateRepository = stateRepository; + this.instanceId = instanceId; + } + + /** + * {@inheritDoc} + */ + @Override + public Trigger save(Trigger trigger) { + observe(trigger); + return super.save(trigger); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean saveIfAbsent( + Trigger trigger) { + observe(trigger); + return super.saveIfAbsent(trigger); + } + + /** + * 记录首次稳定入口保存时的状态。 + * + * @param trigger 待保存触发器 + */ + private void observe(Trigger trigger) { + if (statusObservedAtFirstSave == null) { + ChainState state = stateRepository.load(instanceId); + statusObservedAtFirstSave = + state == null ? null : state.getStatus(); + } + lastTriggerId = trigger.getId(); + } + } + + /** + * 首次保存成功后抛错,用于模拟状态提交前进程退出。 + */ + private static final class FailAfterSaveTriggerStore + extends InMemoryTriggerStore { + + private boolean fail = true; + private String lastTriggerId; + + /** + * {@inheritDoc} + */ + @Override + public Trigger save(Trigger trigger) { + Trigger saved = super.save(trigger); + failAfterFirstSave(trigger); + return saved; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean saveIfAbsent( + Trigger trigger) { + boolean saved = + super.saveIfAbsent(trigger); + if (saved) { + failAfterFirstSave(trigger); + } + return saved; + } + + /** + * 首次成功保存后抛出模拟崩溃。 + * + * @param trigger 已保存触发器 + */ + private void failAfterFirstSave( + Trigger trigger) { + lastTriggerId = trigger.getId(); + if (fail) { + fail = false; + throw new SimulatedCrashException(); + } + } + } + + /** + * 测试专用崩溃异常。 + */ + private static final class SimulatedCrashException + extends RuntimeException { + private static final long serialVersionUID = 1L; + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainTemplateContextTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainTemplateContextTest.java index 04a1e0e..d29f8ec 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainTemplateContextTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainTemplateContextTest.java @@ -1,13 +1,19 @@ package com.easyagents.flow.core.test; +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.chain.RefType; +import com.easyagents.flow.core.chain.repository.InMemoryLoopResultRepository; +import com.easyagents.flow.core.chain.repository.LoopInputReference; import com.easyagents.flow.core.node.StartNode; import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -32,4 +38,229 @@ public class ChainTemplateContextTest { Assert.assertEquals("7", result.get("nextValue")); } + + /** + * 验证审计参数快照保留直接大型引用,不在工作流线程读取完整输入。 + * + * @throws Exception 线程上下文反射失败时抛出 + */ + @Test + public void shouldPreserveDirectReferenceForAuditWithoutLoadingInput() + throws Exception { + CountingLoopRepository repository = + new CountingLoopRepository(); + Chain chain = new Chain( + new ChainDefinition(), + "instance-audit"); + chain.setLoopResultRepository(repository); + Field currentChain = Chain.class + .getDeclaredField( + "EXECUTION_THREAD_LOCAL"); + currentChain.setAccessible(true); + @SuppressWarnings("unchecked") + ThreadLocal context = + (ThreadLocal) currentChain.get(null); + LoopInputReference reference = + new LoopInputReference( + "instance-audit:dataset", + 100_000); + ChainState state = new ChainState(); + state.getMemory().put( + "dataset.data", reference); + Parameter parameter = new Parameter(); + parameter.setName("items"); + parameter.setRefType(RefType.REF); + parameter.setRef("dataset.data"); + StartNode node = new StartNode(); + node.setParameters(List.of(parameter)); + + context.set(chain); + try { + Map result = + state.resolveParametersPreservingReferences( + node); + + Assert.assertSame( + reference, result.get("items")); + Assert.assertEquals( + 0, repository.loadInputCalls); + } finally { + context.remove(); + } + } + + /** + * 验证无关固定参数不会触发 memory 中大型引用的物化。 + * + * @throws Exception 线程上下文反射失败时抛出 + */ + @Test + public void shouldNotLoadUnrelatedReferenceForFixedAuditParameter() + throws Exception { + CountingLoopRepository repository = + new CountingLoopRepository(); + Chain chain = new Chain( + new ChainDefinition(), + "instance-fixed-audit"); + chain.setLoopResultRepository(repository); + Field currentChain = Chain.class + .getDeclaredField( + "EXECUTION_THREAD_LOCAL"); + currentChain.setAccessible(true); + @SuppressWarnings("unchecked") + ThreadLocal context = + (ThreadLocal) currentChain.get(null); + ChainState state = new ChainState(); + state.getMemory().put( + "dataset.data", + new LoopInputReference( + "instance-fixed-audit:dataset", + 100_000)); + state.getMemory().put( + "small.value", "ok"); + Parameter parameter = new Parameter(); + parameter.setName("description"); + parameter.setRefType(RefType.FIXED); + parameter.setValue("{{small.value}}"); + StartNode node = new StartNode(); + node.setParameters(List.of(parameter)); + + context.set(chain); + try { + Map result = + state.resolveParametersPreservingReferences( + node); + + Assert.assertEquals( + "ok", + result.get("description")); + Assert.assertEquals( + 0, repository.loadInputCalls); + } finally { + context.remove(); + } + } + + /** + * 验证普通节点固定参数同样只还原模板实际读取的大型引用。 + * + * @throws Exception 线程上下文反射失败时抛出 + */ + @Test + public void shouldNotLoadUnrelatedReferenceForNormalFixedParameter() + throws Exception { + CountingLoopRepository repository = + new CountingLoopRepository(); + Chain chain = new Chain( + new ChainDefinition(), + "instance-fixed-normal"); + chain.setLoopResultRepository(repository); + Field currentChain = Chain.class + .getDeclaredField( + "EXECUTION_THREAD_LOCAL"); + currentChain.setAccessible(true); + @SuppressWarnings("unchecked") + ThreadLocal context = + (ThreadLocal) currentChain.get(null); + ChainState state = new ChainState(); + state.getMemory().put( + "dataset.data", + new LoopInputReference( + "instance-fixed-normal:dataset", + 100_000)); + state.getMemory().put( + "small.value", "ok"); + Parameter parameter = new Parameter(); + parameter.setName("description"); + parameter.setRefType(RefType.FIXED); + parameter.setValue("{{small.value}}"); + StartNode node = new StartNode(); + node.setParameters(List.of(parameter)); + + context.set(chain); + try { + Map result = + state.resolveParameters(node); + + Assert.assertEquals( + "ok", + result.get("description")); + Assert.assertEquals( + 0, repository.loadInputCalls); + } finally { + context.remove(); + } + } + + /** + * 验证同一模板重复读取一个大型引用时只执行一次仓储还原。 + * + * @throws Exception 线程上下文反射失败时抛出 + */ + @Test + public void shouldMemoizeRepeatedReferenceWithinOneTemplate() + throws Exception { + CountingLoopRepository repository = + new CountingLoopRepository(); + String resultId = + "instance-fixed-repeat:dataset"; + repository.storeInput( + resultId, List.of("item")); + Chain chain = new Chain( + new ChainDefinition(), + "instance-fixed-repeat"); + chain.setLoopResultRepository(repository); + Field currentChain = Chain.class + .getDeclaredField( + "EXECUTION_THREAD_LOCAL"); + currentChain.setAccessible(true); + @SuppressWarnings("unchecked") + ThreadLocal context = + (ThreadLocal) currentChain.get(null); + ChainState state = new ChainState(); + state.getMemory().put( + "dataset.data", + new LoopInputReference( + resultId, 1)); + Parameter parameter = new Parameter(); + parameter.setName("description"); + parameter.setRefType(RefType.FIXED); + parameter.setValue( + "{{dataset.data}}/{{dataset.data}}"); + StartNode node = new StartNode(); + node.setParameters(List.of(parameter)); + + context.set(chain); + try { + Map result = + state.resolveParameters(node); + + Assert.assertEquals( + "[item]/[item]", + result.get("description")); + Assert.assertEquals( + 1, repository.loadInputCalls); + } finally { + context.remove(); + } + } + + /** + * 记录大型输入加载次数的仓储。 + */ + private static final class CountingLoopRepository + extends InMemoryLoopResultRepository { + + private int loadInputCalls; + + /** + * {@inheritDoc} + */ + @Override + public List loadInput( + LoopInputReference reference) { + loadInputCalls++; + return super.loadInput(reference); + } + } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ExecutionBudgetTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ExecutionBudgetTest.java new file mode 100644 index 0000000..70274d6 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ExecutionBudgetTest.java @@ -0,0 +1,63 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.runtime.ExecutionBudget; +import com.easyagents.flow.core.chain.runtime.ExecutionBudgetExceededException; +import org.junit.Assert; +import org.junit.Test; + +/** + * 验证工作流执行预算的宽松默认值和边界检查。 + */ +public class ExecutionBudgetTest { + + /** + * 验证缺省预算与 XL12 约定一致。 + */ + @Test + public void shouldUseGenerousDefaultBudgets() { + ExecutionBudget budget = ExecutionBudget.defaults(); + + Assert.assertEquals(100_000L, budget.getMaxIterations()); + // 人工确认和长期挂起不应被墙钟时间误伤,时长预算默认关闭。 + Assert.assertEquals(0L, budget.getMaxDurationMillis()); + Assert.assertEquals(1_000_000L, budget.getMaxChildExecutions()); + Assert.assertEquals(512L * 1024L * 1024L, budget.getMaxAccumulatedBytes()); + Assert.assertEquals(32, budget.getMaxNestedDepth()); + // 热状态估算可能误伤既有业务,默认关闭并允许部署方显式配置。 + Assert.assertEquals(0L, budget.getMaxHotStateBytes()); + } + + /** + * 验证达到阈值时仍允许执行,超过阈值后才阻止。 + */ + @Test + public void shouldRejectOnlyAfterBudgetIsExceeded() { + ExecutionBudget budget = new ExecutionBudget(3, 1000, 5, 10, 2, 20); + + budget.checkIterations("loop", 3); + budget.checkDuration(1000, 2000); + budget.checkChildExecutions(5); + budget.checkAccumulatedBytes("loop", 10); + budget.checkNestedDepth("loop", 2); + + assertExceeded(() -> budget.checkIterations("loop", 4)); + assertExceeded(() -> budget.checkDuration(1000, 2001)); + assertExceeded(() -> budget.checkChildExecutions(6)); + assertExceeded(() -> budget.checkAccumulatedBytes("loop", 11)); + assertExceeded(() -> budget.checkNestedDepth("loop", 3)); + } + + /** + * 断言动作触发预算超限异常。 + * + * @param action 待执行动作 + */ + private void assertExceeded(Runnable action) { + try { + action.run(); + Assert.fail("Expected ExecutionBudgetExceededException"); + } catch (ExecutionBudgetExceededException expected) { + Assert.assertNotNull(expected.getMessage()); + } + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/GenericNodeLoopCountTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/GenericNodeLoopCountTest.java new file mode 100644 index 0000000..1c2838e --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/GenericNodeLoopCountTest.java @@ -0,0 +1,325 @@ +package com.easyagents.flow.core.test; + +import com.alibaba.fastjson.JSONObject; +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.Edge; +import com.easyagents.flow.core.chain.Node; +import com.easyagents.flow.core.chain.Parameter; +import com.easyagents.flow.core.chain.RefType; +import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import com.easyagents.flow.core.node.BaseNode; +import com.easyagents.flow.core.node.EndNode; +import com.easyagents.flow.core.node.LoopNode; +import com.easyagents.flow.core.node.StartNode; +import com.easyagents.flow.core.parser.ChainParser; +import com.easyagents.flow.core.parser.impl.EndNodeParser; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +/** + * 验证普通节点循环的总次数语义和嵌套计数隔离。 + */ +public class GenericNodeLoopCountTest { + + /** + * 验证默认次数为 1,且零值和超过 300 的值均被拒绝。 + */ + @Test + public void shouldValidateConfiguredLoopCount() { + ProbeNode node = new ProbeNode(); + + Assert.assertEquals(Node.MIN_LOOP_COUNT, node.getMaxLoopCount()); + assertInvalidLoopCount(node, 0); + assertInvalidLoopCount(node, Node.MAX_LOOP_COUNT + 1); + } + + /** + * 验证 JSON 解析缺省为 1,并拒绝零值、小数和超过上限的配置。 + */ + @Test + public void shouldValidateParsedLoopCount() { + ChainParser parser = ChainParser.builder() + .withDefaultParsers(true) + .build(); + Node defaultNode = parseLoopNodeConfiguration(parser, null); + + Assert.assertEquals( + Node.MIN_LOOP_COUNT, + defaultNode.getMaxLoopCount()); + assertInvalidParsedLoopCount(parser, "0"); + assertInvalidParsedLoopCount(parser, "1.5"); + assertInvalidParsedLoopCount( + parser, + String.valueOf(Node.MAX_LOOP_COUNT + 1)); + } + + /** + * 验证填写 1 执行 1 次,填写 3 执行 3 次。 + */ + @Test + public void shouldExecuteConfiguredTotalCount() { + Map once = createExecutor( + createGenericDefinition("generic-once", 1)) + .execute("generic-once", Collections.emptyMap()); + Map threeTimes = createExecutor( + createGenericDefinition("generic-three", 3)) + .execute("generic-three", Collections.emptyMap()); + + Assert.assertEquals(0, once.get("loopIndex")); + Assert.assertEquals(2, threeTimes.get("loopIndex")); + } + + /** + * 验证显式外层循环每一轮都会让普通内层节点从索引 0 重新开始。 + */ + @Test + public void shouldResetInnerGenericLoopForEveryOuterIteration() { + ChainExecutor executor = createExecutor(createNestedDefinition()); + + Map result = executor.execute( + "nested-loop-count", + Collections.singletonMap("times", 3)); + + Assert.assertEquals( + Arrays.asList(1, 1, 1), + result.get("loopIndexes")); + } + + /** + * 创建普通循环定义。 + * + * @param id 定义 ID + * @param loopCount 总执行次数 + * @return 工作流定义 + */ + private ChainDefinition createGenericDefinition( + String id, int loopCount) { + ChainDefinition definition = new ChainDefinition(); + definition.setId(id); + + StartNode start = new StartNode(); + start.setId("start"); + + ProbeNode probe = new ProbeNode(); + probe.setId("probe"); + probe.setLoopEnable(true); + probe.setLoopIntervalMs(0L); + probe.setMaxLoopCount(loopCount); + + EndNode end = new EndNode(); + end.setId("end"); + Parameter output = new Parameter(); + output.setName("loopIndex"); + output.setRef("probe.loopIndex"); + output.setRefType(RefType.REF); + end.setOutputDefs(Collections.singletonList(output)); + + definition.addNode(start); + definition.addNode(probe); + definition.addNode(end); + definition.addEdge(edge("e1", "start", "probe")); + definition.addEdge(edge("e2", "probe", "end")); + return definition; + } + + /** + * 创建显式外层循环嵌套普通内层循环的定义。 + * + * @return 工作流定义 + */ + private ChainDefinition createNestedDefinition() { + ChainDefinition definition = new ChainDefinition(); + definition.setId("nested-loop-count"); + + StartNode start = new StartNode(); + start.setId("start"); + start.setParameters(Collections.singletonList(inputParameter("times"))); + + LoopNode loop = new LoopNode(); + loop.setId("loop"); + Parameter loopVar = new Parameter(); + loopVar.setName("times"); + loopVar.setRef("times"); + loopVar.setRefType(RefType.REF); + loop.setLoopVar(loopVar); + + ProbeNode probe = new ProbeNode(); + probe.setId("probe"); + probe.setParentId("loop"); + probe.setLoopEnable(true); + probe.setLoopIntervalMs(0L); + probe.setMaxLoopCount(2); + + Parameter loopOutput = new Parameter(); + loopOutput.setName("loopIndex"); + loopOutput.setRef("probe.loopIndex"); + loopOutput.setRefType(RefType.REF); + loop.setOutputDefs(Collections.singletonList(loopOutput)); + + EndNode end = new EndNode(); + end.setId("end"); + Parameter result = new Parameter(); + result.setName("loopIndexes"); + result.setRef("loop.loopIndex"); + result.setRefType(RefType.REF); + end.setOutputDefs(Collections.singletonList(result)); + + definition.addNode(start); + definition.addNode(loop); + definition.addNode(probe); + definition.addNode(end); + definition.addEdge(edge("e1", "start", "loop")); + definition.addEdge(edge("e2", "loop", "probe")); + definition.addEdge(edge("e3", "loop", "end")); + return definition; + } + + /** + * 创建执行器。 + * + * @param definition 工作流定义 + * @return 测试执行器 + */ + private ChainExecutor createExecutor(ChainDefinition definition) { + return new ChainExecutor( + new FixedDefinitionRepository(definition), + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository()); + } + + /** + * 断言设置非法循环次数时抛出参数异常。 + * + * @param node 测试节点 + * @param loopCount 非法次数 + */ + private void assertInvalidLoopCount(Node node, int loopCount) { + try { + node.setMaxLoopCount(loopCount); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("maxLoopCount")); + } + } + + /** + * 断言解析非法循环次数时失败。 + * + * @param parser 工作流解析器 + * @param loopCount 原始次数 JSON 值 + */ + private void assertInvalidParsedLoopCount( + ChainParser parser, String loopCount) { + try { + parseLoopNodeConfiguration(parser, loopCount); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("maxLoopCount")); + } + } + + /** + * 解析一个启用普通循环的结束节点。 + * + * @param parser 工作流解析器 + * @param loopCount 循环次数;为空时省略字段 + * @return 解析后的节点 + */ + private Node parseLoopNodeConfiguration( + ChainParser parser, String loopCount) { + JSONObject data = new JSONObject(); + data.put("loopEnable", true); + if (loopCount != null) { + data.put("maxLoopCount", loopCount); + } + JSONObject nodeJson = new JSONObject(); + nodeJson.put("id", "end"); + nodeJson.put("type", "endNode"); + nodeJson.put("data", data); + return new EndNodeParser().parse( + nodeJson, new JSONObject(), parser); + } + + /** + * 创建输入参数。 + * + * @param name 参数名 + * @return 输入参数 + */ + private Parameter inputParameter(String name) { + Parameter parameter = new Parameter(); + parameter.setName(name); + parameter.setRefType(RefType.INPUT); + parameter.setRequired(true); + return parameter; + } + + /** + * 创建连线。 + * + * @param id 连线 ID + * @param source 源节点 + * @param target 目标节点 + * @return 连线 + */ + private Edge edge(String id, String source, String target) { + Edge edge = new Edge(); + edge.setId(id); + edge.setSource(source); + edge.setTarget(target); + return edge; + } + + /** + * 输出当前普通循环的零基索引。 + */ + private static final class ProbeNode extends BaseNode { + + /** + * 执行探针节点。 + * + * @param chain 当前工作流 + * @return 当前零基循环索引 + */ + @Override + public Map execute(Chain chain) { + return Collections.singletonMap( + "loopIndex", + chain.getNodeState(getId()).getLoopCount()); + } + } + + /** + * 固定返回测试定义的仓储。 + */ + private static final class FixedDefinitionRepository + implements ChainDefinitionRepository { + + private final ChainDefinition definition; + + /** + * 创建定义仓储。 + * + * @param definition 工作流定义 + */ + private FixedDefinitionRepository(ChainDefinition definition) { + this.definition = definition; + } + + /** + * {@inheritDoc} + */ + @Override + public ChainDefinition getChainDefinitionById(String id) { + return definition; + } + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/IoBulkheadTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/IoBulkheadTest.java new file mode 100644 index 0000000..09852a2 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/IoBulkheadTest.java @@ -0,0 +1,82 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.runtime.RetryableTriggerException; +import com.easyagents.flow.core.util.IoBulkhead; +import org.junit.Assert; +import org.junit.Test; + +import java.time.Duration; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * {@link IoBulkhead} 有界并发与可观测性回归测试。 + */ +public class IoBulkheadTest { + + /** + * 验证单目标饱和时快速拒绝,同时其他目标仍可使用剩余全局容量。 + * + * @throws Exception 并发测试执行失败时抛出 + */ + @Test + public void shouldIsolateSaturatedTargetWithoutExhaustingGlobalCapacity() throws Exception { + IoBulkhead bulkhead = new IoBulkhead(2, 1, Duration.ofMillis(50), 4); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (IoBulkhead.Permit first = bulkhead.acquire("http:first"); + IoBulkhead.Permit second = bulkhead.acquire("http:second")) { + Future rejected = executor.submit(() -> bulkhead.acquire("http:first")); + try { + rejected.get(); + Assert.fail("same target should be rejected"); + } catch (ExecutionException exception) { + Assert.assertTrue(exception.getCause() instanceof RetryableTriggerException); + } + + IoBulkhead.Snapshot snapshot = bulkhead.snapshot(); + Assert.assertEquals(2L, snapshot.acquiredCount()); + Assert.assertEquals(1L, snapshot.rejectedCount()); + Assert.assertEquals(2L, snapshot.inFlightCount()); + Assert.assertEquals(0, snapshot.availableGlobalPermits()); + Assert.assertEquals(2, snapshot.trackedTargetCount()); + } finally { + executor.shutdownNow(); + } + + Assert.assertEquals(0L, bulkhead.snapshot().inFlightCount()); + Assert.assertEquals(2, bulkhead.snapshot().availableGlobalPermits()); + } + + /** + * 验证 URL 目标键只保留协议无关的主机和显式端口。 + */ + @Test + public void shouldResolveStableHttpTarget() { + Assert.assertEquals( + "http:example.com:8443", + IoBulkhead.targetForUrl("https://EXAMPLE.com:8443/path?q=1")); + Assert.assertEquals("http:unknown", IoBulkhead.targetForUrl("not a url")); + } + + /** + * 验证高基数目标不会突破目标信号量注册上限。 + */ + @Test + public void shouldKeepTrackedTargetRegistryBounded() { + IoBulkhead bulkhead = new IoBulkhead(4, 2, Duration.ofMillis(50), 2); + + try (IoBulkhead.Permit ignored = bulkhead.acquire("target:first")) { + // 仅触发目标注册。 + } + try (IoBulkhead.Permit ignored = bulkhead.acquire("target:second")) { + // 仅触发目标注册。 + } + try (IoBulkhead.Permit ignored = bulkhead.acquire("target:overflow")) { + // 超额目标统一使用有界的 overflow 信号量。 + } + + Assert.assertEquals(2, bulkhead.snapshot().trackedTargetCount()); + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JsConditionUtilTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JsConditionUtilTest.java new file mode 100644 index 0000000..c6c8e5d --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JsConditionUtilTest.java @@ -0,0 +1,91 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.util.JsConditionUtil; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * GraalVM 条件表达式编译复用与上下文隔离回归测试。 + */ +public class JsConditionUtilTest { + + /** + * 验证相同 Source 在不同 Context 中不会残留上一次变量。 + */ + @Test + public void shouldIsolateVariablesWhenReusingSource() { + Chain chain = chain("js-isolation"); + + Assert.assertTrue(JsConditionUtil.eval( + "value > 10", chain, Map.of("value", 11))); + Assert.assertFalse(JsConditionUtil.eval( + "value > 10", chain, Map.of("value", 9))); + } + + /** + * 验证共享 Engine 和 Source 可被多个隔离 Context 并发使用。 + * + * @throws Exception 等待并发任务被中断时抛出 + */ + @Test + public void shouldEvaluateSharedSourceConcurrently() + throws Exception { + Chain chain = chain("js-concurrent"); + ExecutorService workers = + Executors.newFixedThreadPool(4); + CountDownLatch completed = + new CountDownLatch(20); + AtomicInteger failures = + new AtomicInteger(); + try { + for (int value = 0; value < 20; value++) { + int current = value; + workers.submit(() -> { + try { + boolean result = JsConditionUtil.eval( + "value % 2 === 0", + chain, + Map.of("value", current)); + if (result != (current % 2 == 0)) { + failures.incrementAndGet(); + } + } catch (Throwable error) { + failures.incrementAndGet(); + } finally { + completed.countDown(); + } + }); + } + Assert.assertTrue(completed.await( + 10L, TimeUnit.SECONDS)); + Assert.assertEquals(0, failures.get()); + } finally { + workers.shutdownNow(); + } + } + + /** + * 创建带初始化状态的工作流。 + * + * @param instanceId 实例 ID + * @return 测试工作流 + */ + private Chain chain(String instanceId) { + Chain chain = new Chain( + new ChainDefinition(), instanceId); + chain.setChainStateRepository( + new InMemoryChainStateRepository()); + chain.initializeState(); + return chain; + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LegacySerializationCompatibilityTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LegacySerializationCompatibilityTest.java new file mode 100644 index 0000000..9302a64 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LegacySerializationCompatibilityTest.java @@ -0,0 +1,83 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryLoopResultRepository; +import com.easyagents.flow.core.node.LoopNode; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ObjectInputStream; +import java.io.ObjectStreamClass; +import java.lang.reflect.Method; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +/** + * 校验工作流 Java 序列化状态的滚动升级兼容性。 + */ +public class LegacySerializationCompatibilityTest { + + private static final String LEGACY_LOOP_CONTEXT = + "rO0ABXNyADJjb20uZWFzeWFnZW50cy5mbG93LmNvcmUubm9kZS5Mb29wTm9k" + + "ZSRMb29wQ29udGV4dEj5b620EJczAgACSQAMY3VycmVudEluZGV4TAAJc3Vi" + + "UmVzdWx0dAAPTGphdmEvdXRpbC9NYXA7eHAAAAADc3IAF2phdmEudXRpbC5M" + + "aW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZh" + + "LnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVz" + + "aG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAFdmFsdWVzcgATamF2YS51dGls" + + "LkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAAncEAAAAAnQAAWF0" + + "AAFieHgA"; + + /** + * 保护历史类的默认序列化 UID,避免缓存状态在滚动升级时失效。 + */ + @Test + public void shouldKeepLegacySerialVersionUids() { + Assert.assertEquals(-7958235553581638052L, + ObjectStreamClass.lookup(ChainState.class).getSerialVersionUID()); + Assert.assertEquals(-6727481826462129573L, + ObjectStreamClass.lookup(NodeState.class).getSerialVersionUID()); + Assert.assertEquals(5258356831772776243L, + ObjectStreamClass.lookup(LoopNode.LoopContext.class).getSerialVersionUID()); + } + + /** + * 旧 LoopContext 中的内嵌结果应惰性迁移到分块结果仓储。 + * + * @throws Exception 反序列化或反射调用失败 + */ + @Test + @SuppressWarnings("unchecked") + public void shouldMigrateLegacyLoopResultWithoutDataLoss() throws Exception { + LoopNode.LoopContext context; + byte[] bytes = Base64.getDecoder().decode(LEGACY_LOOP_CONTEXT); + try (ObjectInputStream input = new ObjectInputStream(new ByteArrayInputStream(bytes))) { + context = (LoopNode.LoopContext) input.readObject(); + } + Assert.assertEquals(3, context.getCurrentIndex()); + Assert.assertEquals(List.of("a", "b"), context.getSubResult().get("value")); + Assert.assertNull(context.getResultId()); + + InMemoryLoopResultRepository resultRepository = new InMemoryLoopResultRepository(); + Chain chain = new Chain(new ChainDefinition(), "legacy-chain"); + chain.setChainStateRepository(new InMemoryChainStateRepository()); + chain.setLoopResultRepository(resultRepository); + + LoopNode loopNode = new LoopNode(); + Method migrate = LoopNode.class.getDeclaredMethod( + "migrateLegacyResult", Chain.class, LoopNode.LoopContext.class); + migrate.setAccessible(true); + migrate.invoke(loopNode, chain, context); + + Assert.assertNotNull(context.getResultId()); + Assert.assertNull(context.getSubResult()); + Map migrated = + resultRepository.load(context.getResultId(), 2, List.of("value")); + Assert.assertEquals(List.of("a", "b"), migrated.get("value")); + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java index bfa2969..30ea78d 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java @@ -3,8 +3,18 @@ package com.easyagents.flow.core.test; import com.easyagents.flow.core.chain.*; import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository; import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryLoopResultRepository; import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.chain.repository.LoopInputReference; +import com.easyagents.flow.core.chain.repository.NodeStateField; +import com.easyagents.flow.core.chain.repository.NodeStateRepository; import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import com.easyagents.flow.core.chain.runtime.ExecutionBudget; +import com.easyagents.flow.core.chain.runtime.ExecutionBudgetExceededException; +import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import com.easyagents.flow.core.chain.event.NodeEndEvent; +import com.easyagents.flow.core.chain.event.NodeStartEvent; import com.easyagents.flow.core.node.BaseNode; import com.easyagents.flow.core.node.EndNode; import com.easyagents.flow.core.node.LoopNode; @@ -12,9 +22,19 @@ import com.easyagents.flow.core.node.StartNode; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.Arrays; +import java.util.AbstractList; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.Executors; /** * 验证循环体内可以读取父循环节点上一轮的最新输出。 @@ -23,6 +43,287 @@ public class LoopNodeProgressContextTest { @Test public void shouldExposeLatestLoopOutputInsideLoopBody() { + ChainDefinition definition = createDefinition(); + ChainExecutor executor = createExecutor(definition); + Map variables = new HashMap<>(); + variables.put("times", 2); + + Map resultMap = executor.execute("loop-progress-test", variables); + + Assert.assertEquals(Arrays.asList("1", "2"), resultMap.get("result")); + } + + /** + * 验证循环节点跨子触发器完成时复用同一稳定业务尝试键。 + */ + @Test + public void shouldKeepLoopExecutionAttemptKeyStable() { + ChainDefinition definition = + createDefinition(); + ChainExecutor executor = + createExecutor(definition); + java.util.concurrent.atomic.AtomicReference + startedKey = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference + endedKey = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference + endedStatus = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference + endedError = + new java.util.concurrent.atomic.AtomicReference<>(); + executor.addEventListener( + NodeStartEvent.class, + (event, chain) -> { + NodeStartEvent startEvent = + (NodeStartEvent) event; + if ("loop".equals( + startEvent.getNode().getId())) { + startedKey.set( + startEvent + .getExecutionAttemptKey()); + } + }); + executor.addEventListener( + NodeEndEvent.class, + (event, chain) -> { + NodeEndEvent endEvent = + (NodeEndEvent) event; + if ("loop".equals( + endEvent.getNode().getId())) { + // 模拟下一轮执行已抢先覆盖可变节点状态。 + chain.updateNodeStateSafely( + "loop", + state -> { + state.setExecutionAttemptKey( + "next-attempt"); + state.setStatus( + NodeStatus.RUNNING); + state.setError( + new ExceptionSummary( + new IllegalStateException( + "next-error"))); + return EnumSet.of( + NodeStateField.STATUS, + NodeStateField.ERROR, + NodeStateField + .EXECUTION_ATTEMPT_KEY); + }); + endedKey.set( + endEvent + .getExecutionAttemptKey()); + endedStatus.set( + endEvent.getStatus()); + endedError.set( + endEvent.getError()); + } + }); + + executor.execute( + "loop-attempt-key-test", + Collections.singletonMap( + "times", 3)); + + Assert.assertNotNull(startedKey.get()); + Assert.assertEquals( + startedKey.get(), + endedKey.get()); + Assert.assertEquals( + NodeStatus.SUCCEEDED, + endedStatus.get()); + Assert.assertNull( + endedError.get()); + } + + /** + * 验证普通 Iterable 只物化一次,后续循环不会从头重复遍历。 + */ + @Test + public void shouldMaterializeNonListIterableOnlyOnce() { + ChainDefinition definition = createDefinition(); + ChainExecutor executor = createExecutor(definition); + OneShotIterable iterable = new OneShotIterable(); + Map variables = new HashMap<>(); + variables.put("times", iterable); + + Map resultMap = executor.execute("loop-progress-test", variables); + + Assert.assertEquals(Arrays.asList("1", "2", "3"), resultMap.get("result")); + Assert.assertEquals(1, iterable.getIteratorCount()); + } + + /** + * 验证超过直接索引阈值的 List 首轮物化后不再回读原集合。 + */ + @Test + public void shouldMaterializeListOnlyOnce() { + ChainDefinition definition = createDefinition(); + ChainExecutor executor = createExecutor(definition); + Integer[] values = new Integer[128]; + Arrays.fill(values, 1); + CountingList list = new CountingList(values); + + Map resultMap = executor.execute( + "loop-list-test", Collections.singletonMap("times", list)); + + Assert.assertEquals(128, ((java.util.List) resultMap.get("result")).size()); + Assert.assertTrue( + "list input should not be re-read for every iteration", + list.getReadCount() <= values.length * 5); + } + + /** + * 验证循环直接消费上游已分页物化的轻量引用,不先还原完整列表。 + */ + @Test + public void shouldConsumePreMaterializedInputReference() { + ChainDefinition definition = createDefinition(); + InMemoryLoopResultRepository loopRepository = + new InMemoryLoopResultRepository(); + String resultId = "loop-reference-input"; + loopRepository.storeInput( + resultId, Arrays.asList(10, 20, 30)); + TriggerScheduler scheduler = + new TriggerScheduler( + new InMemoryTriggerStore(), + Executors.newScheduledThreadPool(2), + Executors.newFixedThreadPool(2), + 1_000L); + try { + ChainExecutor executor = new ChainExecutor( + new FixedDefinitionRepository( + definition), + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository(), + loopRepository, + scheduler, + ExecutionBudget.defaults()); + + Map resultMap = + executor.execute( + "loop-reference-test", + Collections.singletonMap( + "times", + new LoopInputReference( + resultId, 3))); + + Assert.assertEquals( + Arrays.asList("1", "2", "3"), + resultMap.get("result")); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证对象数组和基础类型数组均可作为循环输入。 + */ + @Test + public void shouldSupportArrayInputs() { + ChainDefinition definition = createDefinition(); + ChainExecutor executor = createExecutor(definition); + + Map objectArrayResult = executor.execute( + "loop-object-array-test", + Collections.singletonMap("times", new String[]{"a", "b"})); + Map primitiveArrayResult = executor.execute( + "loop-primitive-array-test", + Collections.singletonMap("times", new int[]{1, 2, 3})); + + Assert.assertEquals(Arrays.asList("1", "2"), objectArrayResult.get("result")); + Assert.assertEquals( + Arrays.asList("1", "2", "3"), primitiveArrayResult.get("result")); + } + + /** + * 验证显式循环节点接受 300 次,并拒绝 301 次的数值输入。 + */ + @Test + public void shouldEnforceNumericLoopLimit() { + ChainExecutor maximumExecutor = createExecutor(createDefinition()); + Map maximumResult = maximumExecutor.execute( + "loop-maximum-test", + Collections.singletonMap("times", Node.MAX_LOOP_COUNT)); + + Assert.assertEquals( + Node.MAX_LOOP_COUNT, + ((java.util.List) maximumResult.get("result")).size()); + + ChainExecutor exceededExecutor = createExecutor(createDefinition()); + assertLoopFailure( + exceededExecutor, + "loop-exceeded-test", + Collections.singletonMap( + "times", Node.MAX_LOOP_COUNT + 1), + IllegalArgumentException.class); + } + + /** + * 验证未知大小 Iterable 在物化第 301 个元素前终止。 + */ + @Test + public void shouldStopOversizedIterableDuringMaterialization() { + ChainExecutor executor = createExecutor(createDefinition()); + + assertLoopFailure( + executor, + "loop-iterable-exceeded-test", + Collections.singletonMap( + "times", + new RangeIterable( + Node.MAX_LOOP_COUNT + 1)), + ExecutionBudgetExceededException.class); + } + + /** + * 验证大循环累计结果不会进入每轮持久化的节点热状态。 + */ + @Test + public void shouldKeepNodeStateCompactWhenCollectingLargeLoopResult() { + ChainDefinition definition = createDefinition(); + TrackingNodeStateRepository nodeStateRepository = new TrackingNodeStateRepository(); + ChainExecutor executor = new ChainExecutor(new FixedDefinitionRepository(definition), + new InMemoryChainStateRepository(), + nodeStateRepository); + Map variables = new HashMap<>(); + variables.put("times", Node.MAX_LOOP_COUNT); + + Map resultMap = executor.execute("loop-progress-test", variables); + + @SuppressWarnings("unchecked") + java.util.List result = (java.util.List) resultMap.get("result"); + Assert.assertEquals(Node.MAX_LOOP_COUNT, result.size()); + Assert.assertEquals("1", result.get(0)); + Assert.assertEquals( + String.valueOf(Node.MAX_LOOP_COUNT), + result.get(Node.MAX_LOOP_COUNT - 1)); + Assert.assertTrue("loop node hot state should remain bounded", + nodeStateRepository.getMaxSerializedBytes() < 16 * 1024); + } + + /** + * 验证循环体存在多个直属分支时,必须等待全部分支完成后再推进下一轮。 + */ + @Test + public void shouldWaitForAllDirectLoopBranchesBeforeAdvancing() { + ChainDefinition definition = createMultiBranchDefinition(); + ChainExecutor executor = createExecutor(definition); + + Map resultMap = executor.execute( + "loop-multi-branch-test", Collections.singletonMap("times", 3)); + + Assert.assertEquals(Arrays.asList("a0", "a1", "a2"), resultMap.get("a")); + Assert.assertEquals(Arrays.asList("b0", "b1", "b2"), resultMap.get("b")); + } + + /** + * 创建循环进度测试定义。 + * + * @return 工作流定义 + */ + private ChainDefinition createDefinition() { ChainDefinition definition = new ChainDefinition(); definition.setId("loop-progress-test"); @@ -67,17 +368,115 @@ public class LoopNodeProgressContextTest { definition.addEdge(edge("e1", "start", "loop")); definition.addEdge(edge("e2", "loop", "acc")); definition.addEdge(edge("e3", "loop", "end")); + return definition; + } - ChainExecutor executor = new ChainExecutor(new FixedDefinitionRepository(definition), + /** + * 创建包含两个并行直属循环分支的定义。 + * + * @return 工作流定义 + */ + private ChainDefinition createMultiBranchDefinition() { + ChainDefinition definition = new ChainDefinition(); + definition.setId("loop-multi-branch-test"); + + StartNode startNode = new StartNode(); + startNode.setId("start"); + startNode.setParameters(Collections.singletonList(inputParameter("times"))); + + LoopNode loopNode = new LoopNode(); + loopNode.setId("loop"); + Parameter loopVar = new Parameter(); + loopVar.setName("times"); + loopVar.setRef("times"); + loopVar.setRefType(RefType.REF); + loopNode.setLoopVar(loopVar); + + BranchNode branchA = new BranchNode("a", 0L); + branchA.setId("branch-a"); + branchA.setParentId("loop"); + BranchNode branchB = new BranchNode("b", 20L); + branchB.setId("branch-b"); + branchB.setParentId("loop"); + + Parameter outputA = new Parameter(); + outputA.setName("a"); + outputA.setRef("branch-a.value"); + outputA.setRefType(RefType.REF); + Parameter outputB = new Parameter(); + outputB.setName("b"); + outputB.setRef("branch-b.value"); + outputB.setRefType(RefType.REF); + loopNode.setOutputDefs(Arrays.asList(outputA, outputB)); + + EndNode endNode = new EndNode(); + endNode.setId("end"); + Parameter resultA = new Parameter(); + resultA.setName("a"); + resultA.setRef("loop.a"); + resultA.setRefType(RefType.REF); + Parameter resultB = new Parameter(); + resultB.setName("b"); + resultB.setRef("loop.b"); + resultB.setRefType(RefType.REF); + endNode.setOutputDefs(Arrays.asList(resultA, resultB)); + + definition.addNode(startNode); + definition.addNode(loopNode); + definition.addNode(branchA); + definition.addNode(branchB); + definition.addNode(endNode); + definition.addEdge(edge("e1", "start", "loop")); + definition.addEdge(edge("e2", "loop", "branch-a")); + definition.addEdge(edge("e3", "loop", "branch-b")); + definition.addEdge(edge("e4", "loop", "end")); + return definition; + } + + /** + * 创建使用内存状态仓储的测试执行器。 + * + * @param definition 工作流定义 + * @return 测试执行器 + */ + private ChainExecutor createExecutor(ChainDefinition definition) { + return new ChainExecutor(new FixedDefinitionRepository(definition), new InMemoryChainStateRepository(), new InMemoryNodeStateRepository()); + } - Map variables = new HashMap<>(); - variables.put("times", 2); - - Map resultMap = executor.execute("loop-progress-test", variables); - - Assert.assertEquals(java.util.Arrays.asList("1", "2"), resultMap.get("result")); + /** + * 断言循环节点通过结束事件报告指定异常类型。 + * + * @param executor 测试执行器 + * @param definitionId 定义 ID + * @param variables 输入变量 + * @param expectedType 期望异常类型 + */ + private static void assertLoopFailure( + ChainExecutor executor, + String definitionId, + Map variables, + Class expectedType) { + AtomicReference nodeError = new AtomicReference<>(); + executor.addEventListener( + NodeEndEvent.class, + (event, chain) -> { + NodeEndEvent endEvent = (NodeEndEvent) event; + if ("loop".equals(endEvent.getNode().getId()) + && endEvent.getError() != null) { + nodeError.set(endEvent.getError()); + } + }); + try { + executor.execute(definitionId, variables); + Assert.fail("Expected exception: " + expectedType.getSimpleName()); + } catch (RuntimeException expected) { + Assert.assertNotNull("Loop node error event is missing", nodeError.get()); + Assert.assertTrue( + "Unexpected loop node error: " + nodeError.get(), + expectedType.isInstance(nodeError.get())); + } } private static Parameter inputParameter(String name) { @@ -109,6 +508,171 @@ public class LoopNodeProgressContextTest { } } + /** + * 第二次获取迭代器时直接失败,用于识别重复遍历。 + */ + private static class OneShotIterable implements Iterable { + private final AtomicInteger iteratorCount = new AtomicInteger(); + + /** + * 获取唯一可用的迭代器。 + * + * @return 测试迭代器 + */ + @Override + public Iterator iterator() { + if (iteratorCount.incrementAndGet() > 1) { + throw new IllegalStateException("Iterable must not be traversed more than once"); + } + return Arrays.asList(10, 20, 30).iterator(); + } + + /** + * 获取迭代器创建次数。 + * + * @return 迭代器创建次数 + */ + private int getIteratorCount() { + return iteratorCount.get(); + } + } + + /** + * 生成指定数量元素且无法提前获知大小的 Iterable。 + */ + private static final class RangeIterable implements Iterable { + + private final int itemCount; + + /** + * 创建范围输入。 + * + * @param itemCount 元素数量 + */ + private RangeIterable(int itemCount) { + this.itemCount = itemCount; + } + + /** + * {@inheritDoc} + */ + @Override + public Iterator iterator() { + return new Iterator() { + private int index; + + @Override + public boolean hasNext() { + return index < itemCount; + } + + @Override + public Integer next() { + if (!hasNext()) { + throw new java.util.NoSuchElementException(); + } + return index++; + } + }; + } + } + + /** + * 记录元素读取次数的列表。 + */ + private static final class CountingList extends AbstractList { + + private final java.util.List values; + private final AtomicInteger readCount = new AtomicInteger(); + + /** + * 创建计数列表。 + * + * @param values 列表元素 + */ + private CountingList(Integer... values) { + this.values = Arrays.asList(values); + } + + /** + * {@inheritDoc} + */ + @Override + public Integer get(int index) { + readCount.incrementAndGet(); + return values.get(index); + } + + /** + * {@inheritDoc} + */ + @Override + public int size() { + return values.size(); + } + + /** + * 获取元素读取次数。 + * + * @return 元素读取次数 + */ + private int getReadCount() { + return readCount.get(); + } + } + + /** + * 记录节点状态序列化体积的测试仓储。 + */ + private static final class TrackingNodeStateRepository implements NodeStateRepository { + + private final InMemoryNodeStateRepository delegate = new InMemoryNodeStateRepository(); + private final AtomicInteger maxSerializedBytes = new AtomicInteger(); + + /** + * {@inheritDoc} + */ + @Override + public NodeState load(String instanceId, String nodeId) { + return delegate.load(instanceId, nodeId); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean tryUpdate(NodeState newState, EnumSet fields, long chainStateVersion) { + maxSerializedBytes.accumulateAndGet(serializedSize(newState), Math::max); + return delegate.tryUpdate(newState, fields, chainStateVersion); + } + + /** + * 获取观测到的最大序列化字节数。 + * + * @return 最大序列化字节数 + */ + private int getMaxSerializedBytes() { + return maxSerializedBytes.get(); + } + + /** + * 计算节点状态的 Java 序列化字节数。 + * + * @param state 节点状态 + * @return 序列化字节数 + */ + private int serializedSize(NodeState state) { + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + ObjectOutputStream objectOutput = new ObjectOutputStream(output)) { + objectOutput.writeObject(state); + objectOutput.flush(); + return output.size(); + } catch (IOException error) { + throw new IllegalStateException("Failed to serialize node state", error); + } + } + } + /** * 每一轮都读取父循环节点上一轮的 current,再计算新的结果。 */ @@ -142,4 +706,31 @@ public class LoopNodeProgressContextTest { return result; } } + + /** + * 输出当前循环序号的测试分支。 + */ + private static class BranchNode extends BaseNode { + private final String prefix; + private final long delayMs; + + private BranchNode(String prefix, long delayMs) { + this.prefix = prefix; + this.delayMs = delayMs; + } + + @Override + public Map execute(Chain chain) { + if (delayMs > 0) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Branch execution interrupted", error); + } + } + Object index = chain.getState().resolveValue("loop.index"); + return Collections.singletonMap("value", prefix + index); + } + } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java new file mode 100644 index 0000000..b60318a --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java @@ -0,0 +1,123 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.repository.InMemoryLoopResultRepository; +import com.easyagents.flow.core.chain.repository.LoopInputReference; +import com.easyagents.flow.core.chain.repository.LoopResultReference; +import com.easyagents.flow.core.chain.runtime.ExecutionBudgetExceededException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 循环结果引用批量还原回归测试。 + */ +public class LoopResultReferenceResolverTest { + + /** + * 验证同一循环的多个输出只触发一次仓储加载。 + */ + @Test + @SuppressWarnings("unchecked") + public void shouldBatchOutputsFromSameLoopResult() { + CountingLoopResultRepository repository = new CountingLoopResultRepository(); + repository.append("result-1", 0, Map.of("a", "a0", "b", "b0")); + repository.append("result-1", 1, Map.of("a", "a1", "b", "b1")); + Map value = new LinkedHashMap<>(); + value.put("a", new LoopResultReference("result-1", 2, "a")); + value.put("nested", List.of(new LoopResultReference("result-1", 2, "b"))); + + Map resolved = + (Map) repository.resolveReferences(value); + + Assert.assertEquals(Arrays.asList("a0", "a1"), resolved.get("a")); + Assert.assertEquals( + Arrays.asList("b0", "b1"), + ((List) resolved.get("nested")).get(0)); + Assert.assertEquals(1, repository.getLoadCount()); + } + + /** + * 验证无限 Iterable 在达到预算后立即停止物化。 + */ + @Test + public void shouldStopInfiniteInputAtIterationBudget() { + InMemoryLoopResultRepository repository = new InMemoryLoopResultRepository(); + AtomicInteger reads = new AtomicInteger(); + Iterable infinite = () -> new Iterator() { + @Override + public boolean hasNext() { + return true; + } + + @Override + public Integer next() { + return reads.incrementAndGet(); + } + }; + + try { + repository.storeInput("bounded-input", infinite, 3L); + Assert.fail("infinite input should exceed the iteration budget"); + } catch (ExecutionBudgetExceededException expected) { + Assert.assertTrue(expected.getMessage().contains("more than 3")); + } + + Assert.assertEquals(3, reads.get()); + } + + /** + * 验证外置循环输入在业务读取边界仍还原为原顺序列表。 + */ + @Test + public void shouldResolveExternalizedLoopInput() { + InMemoryLoopResultRepository repository = + new InMemoryLoopResultRepository(); + String resultId = "input-reference"; + Assert.assertEquals( + 3, + repository.storeInput( + resultId, + List.of("first", "second", "third"))); + + Object resolved = repository.resolveReferences( + new LoopInputReference(resultId, 3)); + + Assert.assertEquals( + List.of("first", "second", "third"), + resolved); + } + + /** + * 记录加载次数的循环结果仓储。 + */ + private static final class CountingLoopResultRepository + extends InMemoryLoopResultRepository { + + private final AtomicInteger loadCount = new AtomicInteger(); + + /** + * {@inheritDoc} + */ + @Override + public Map load( + String resultId, int iterationCount, List outputNames) { + loadCount.incrementAndGet(); + return super.load(resultId, iterationCount, outputNames); + } + + /** + * 获取仓储加载次数。 + * + * @return 加载次数 + */ + private int getLoadCount() { + return loadCount.get(); + } + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/OkHttpClientUtilTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/OkHttpClientUtilTest.java new file mode 100644 index 0000000..ecdb2df --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/OkHttpClientUtilTest.java @@ -0,0 +1,116 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.util.IoBulkhead; +import com.easyagents.flow.core.util.OkHttpClientUtil; +import com.sun.net.httpserver.HttpServer; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.Assert; +import org.junit.Test; + +import java.net.InetSocketAddress; +import java.time.Duration; + +/** + * {@link OkHttpClientUtil} 共享连接池回归测试。 + */ +public class OkHttpClientUtilTest { + + /** + * 验证默认客户端跨节点调用复用同一实例。 + */ + @Test + public void shouldReuseDefaultClientInstance() { + OkHttpClient first = OkHttpClientUtil.buildDefaultClient(); + OkHttpClient second = OkHttpClientUtil.buildDefaultClient(); + + Assert.assertSame(first, second); + Assert.assertSame(first.connectionPool(), second.connectionPool()); + Assert.assertSame(first.dispatcher(), second.dispatcher()); + Assert.assertEquals( + 1L, + first.interceptors().stream() + .filter(interceptor -> interceptor.getClass().getSimpleName() + .equals("IoBulkheadInterceptor")) + .count()); + } + + /** + * 验证非幂等请求客户端关闭底层隐式重试且继续复用连接资源。 + */ + @Test + public void shouldReuseNoRetryClientAndConnectionPool() { + OkHttpClient defaultClient = OkHttpClientUtil.buildDefaultClient(); + OkHttpClient first = OkHttpClientUtil.buildNoRetryClient(); + OkHttpClient second = OkHttpClientUtil.buildNoRetryClient(); + + Assert.assertSame(first, second); + Assert.assertFalse(first.retryOnConnectionFailure()); + Assert.assertSame(defaultClient.connectionPool(), first.connectionPool()); + Assert.assertSame(defaultClient.dispatcher(), first.dispatcher()); + } + + /** + * 验证单目标并发为一时一次 HTTP 请求只申请一次许可且不会自锁。 + * + * @throws Exception 启动本地 HTTP 服务或请求失败时抛出 + */ + @Test + public void shouldAcquireOnePermitPerHttpRequest() throws Exception { + IoBulkhead.Settings restricted = + new IoBulkhead.Settings(1, 1, Duration.ofMillis(200), 16); + IoBulkhead.configure( + restricted, + new IoBulkhead.Settings(32, 8, Duration.ofSeconds(1), 512), + new IoBulkhead.Settings(24, 12, Duration.ofSeconds(2), 256), + new IoBulkhead.Settings(8, 4, Duration.ofSeconds(2), 128), + new IoBulkhead.Settings(8, 4, Duration.ofSeconds(2), 1_024)); + HttpServer server = HttpServer.create( + new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + byte[] body = "ok".getBytes(java.nio.charset.StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + try { + long acquiredBefore = IoBulkhead.shared() + .snapshot() + .acquiredCount(); + Request request = new Request.Builder() + .url("http://127.0.0.1:" + + server.getAddress().getPort() + + "/") + .build(); + try (Response response = OkHttpClientUtil + .buildNoRetryClient() + .newCall(request) + .execute()) { + Assert.assertEquals(200, response.code()); + } + Assert.assertEquals( + acquiredBefore + 1, + IoBulkhead.shared().snapshot().acquiredCount()); + Assert.assertEquals( + 0L, + IoBulkhead.shared().snapshot().inFlightCount()); + } finally { + server.stop(0); + restoreDefaultBulkheads(); + } + } + + /** + * 恢复测试进程的宽松默认隔离配置,避免污染其他测试。 + */ + private void restoreDefaultBulkheads() { + IoBulkhead.configure( + new IoBulkhead.Settings(64, 16, Duration.ofSeconds(1), 1_024), + new IoBulkhead.Settings(32, 8, Duration.ofSeconds(1), 512), + new IoBulkhead.Settings(24, 12, Duration.ofSeconds(2), 256), + new IoBulkhead.Settings(8, 4, Duration.ofSeconds(2), 128), + new IoBulkhead.Settings(8, 4, Duration.ofSeconds(2), 1_024)); + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TextTemplatePathTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TextTemplatePathTest.java index 38f98f6..e2f0c3f 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TextTemplatePathTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TextTemplatePathTest.java @@ -44,4 +44,66 @@ public class TextTemplatePathTest { Assert.assertEquals("value=", result); } + + /** + * 验证分层上下文保持后层覆盖前层的原有优先级。 + */ + @Test + public void shouldResolveLayeredContextUsingLastMapPrecedence() { + Map memory = new HashMap<>(); + memory.put("name", "memory"); + Map parameters = new HashMap<>(); + parameters.put("name", "parameter"); + + String result = TextTemplate.of("{{name}}") + .formatToString(Arrays.asList(memory, parameters)); + + Assert.assertEquals("parameter", result); + } + + /** + * 验证分层上下文支持跨层兜底并保持 JSON 转义。 + */ + @Test + public void shouldResolveFallbackAcrossLayersWithJsonEscaping() { + Map memory = new HashMap<>(); + memory.put("fallback", "a\"b"); + + String result = TextTemplate.of("{\"value\":\"{{missing ?? fallback}}\"}") + .formatToString(Arrays.asList(memory, Collections.emptyMap()), true); + + Assert.assertEquals("{\"value\":\"a\\\"b\"}", result); + } + + /** + * 后层拥有顶级键时,空对象应完整遮蔽前层同名对象。 + */ + @Test + public void shouldPreservePutAllShadowingForNestedObject() { + Map memory = new HashMap<>(); + memory.put("user", Collections.singletonMap("name", "legacy")); + Map parameters = new HashMap<>(); + parameters.put("user", Collections.emptyMap()); + + String result = TextTemplate.of("{{user.name ?? \"fallback\"}}") + .formatToString(Arrays.asList(memory, parameters)); + + Assert.assertEquals("fallback", result); + } + + /** + * 后层显式 null 应遮蔽前层同名对象并进入模板兜底。 + */ + @Test + public void shouldPreservePutAllShadowingForExplicitNull() { + Map memory = new HashMap<>(); + memory.put("user", Collections.singletonMap("name", "legacy")); + Map parameters = new HashMap<>(); + parameters.put("user", null); + + String result = TextTemplate.of("{{user.name ?? \"fallback\"}}") + .formatToString(Arrays.asList(memory, parameters)); + + Assert.assertEquals("fallback", result); + } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TriggerSchedulerReliabilityTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TriggerSchedulerReliabilityTest.java new file mode 100644 index 0000000..e60bc2d --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/TriggerSchedulerReliabilityTest.java @@ -0,0 +1,973 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; +import com.easyagents.flow.core.chain.runtime.NonRetryableTriggerException; +import com.easyagents.flow.core.chain.runtime.RetryableTriggerException; +import com.easyagents.flow.core.chain.runtime.Trigger; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import com.easyagents.flow.core.chain.runtime.TriggerStore; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.lang.reflect.Field; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * {@link TriggerScheduler} 认领、确认和失败释放语义回归测试。 + */ +public class TriggerSchedulerReliabilityTest { + + /** + * 验证消费者尚未注册时主动触发不会丢失待执行任务。 + */ + @Test + public void shouldKeepTriggerPendingWhenConsumerIsUnavailable() { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + Trigger trigger = futureTrigger("consumer-unavailable"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + try { + Assert.assertFalse(scheduler.fire(trigger.getId())); + Assert.assertNotNull(store.find(trigger.getId())); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证执行失败后触发器被释放,并能再次认领成功。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldReleaseFailedTriggerForRetry() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + Trigger trigger = futureTrigger("retry-after-failure"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + AtomicInteger attempts = new AtomicInteger(); + CountDownLatch firstAttempt = new CountDownLatch(1); + CountDownLatch succeeded = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> { + if (attempts.incrementAndGet() == 1) { + firstAttempt.countDown(); + throw new IllegalStateException("expected first failure"); + } + succeeded.countDown(); + }); + + try { + Assert.assertTrue(scheduler.fire(trigger.getId())); + Assert.assertTrue(firstAttempt.await(2, TimeUnit.SECONDS)); + awaitPending(store, trigger.getId()); + Assert.assertTrue(scheduler.fire(trigger.getId())); + Assert.assertTrue(succeeded.await(2, TimeUnit.SECONDS)); + Assert.assertEquals(2, attempts.get()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证并发认领同一触发器时只有一个消费者获得执行权。 + * + * @throws Exception 并发任务等待失败时抛出 + */ + @Test + public void shouldDispatchClaimedTriggerOnlyOnce() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + Trigger trigger = futureTrigger("single-claim"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + CountDownLatch consumed = new CountDownLatch(1); + AtomicInteger executions = new AtomicInteger(); + scheduler.registerConsumer((current, worker) -> { + executions.incrementAndGet(); + consumed.countDown(); + }); + ExecutorService callers = Executors.newFixedThreadPool(2); + + try { + List> results = callers.invokeAll(Arrays.asList( + () -> scheduler.fire(trigger.getId()), + () -> scheduler.fire(trigger.getId()))); + int accepted = 0; + for (java.util.concurrent.Future result : results) { + if (result.get()) { + accepted++; + } + } + Assert.assertEquals(1, accepted); + Assert.assertTrue(consumed.await(2, TimeUnit.SECONDS)); + Assert.assertEquals(1, executions.get()); + } finally { + callers.shutdownNow(); + scheduler.shutdown(); + } + } + + /** + * 验证缺少认领对象时旧 ID 接口不会静默执行不安全确认。 + */ + @Test + public void shouldRejectClaimMutationWithoutClaimedTrigger() { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + + try { + store.renewClaim("unsafe-id-only", 1000L); + Assert.fail("ID-only renewal must be rejected"); + } catch (UnsupportedOperationException expected) { + Assert.assertTrue(expected.getMessage().contains("token")); + } + + try { + store.acknowledge("unsafe-id-only"); + Assert.fail("ID-only acknowledge must be rejected"); + } catch (UnsupportedOperationException expected) { + Assert.assertTrue(expected.getMessage().contains("trigger")); + } + } + + /** + * 验证不可重试错误会移出待执行集合,避免形成永久热重放。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldDeadLetterNonRetryableTrigger() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + Trigger trigger = futureTrigger("non-retryable"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + CountDownLatch attempted = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> { + attempted.countDown(); + throw new NonRetryableTriggerException("invalid definition"); + }); + + try { + Assert.assertTrue(scheduler.fire(trigger.getId())); + Assert.assertTrue(attempted.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (store.find(trigger.getId()) != null && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertNull(store.find(trigger.getId())); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证瞬时基础设施冲突即使超过普通投递上限也继续保留,避免幂等 owner 存活期内误死信。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldNotDeadLetterTransientContention() + throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + Trigger trigger = futureTrigger("transient-contention"); + trigger.setDeliveryAttempt(100); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + CountDownLatch attempted = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> { + attempted.countDown(); + throw new RetryableTriggerException( + "operation still owned", null); + }); + + try { + Assert.assertTrue(scheduler.fire(trigger.getId())); + Assert.assertTrue(attempted.await(2, TimeUnit.SECONDS)); + awaitPending(store, trigger.getId()); + Assert.assertEquals( + 101, + store.find(trigger.getId()) + .getDeliveryAttempt()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证业务终态尚未持久化时不得先移除触发器。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldRetryTerminalConvergenceBeforeDeadLetter() + throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + Trigger trigger = futureTrigger( + "terminal-convergence-retry"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + AtomicInteger convergenceAttempts = new AtomicInteger(); + AtomicInteger executions = new AtomicInteger(); + CountDownLatch consumed = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> { + executions.incrementAndGet(); + consumed.countDown(); + throw new NonRetryableTriggerException( + "invalid definition"); + }); + scheduler.registerFailureListener((current, failure) -> + convergenceAttempts.incrementAndGet() >= 2); + + try { + Assert.assertTrue(scheduler.fire(trigger.getId())); + awaitPending(store, trigger.getId()); + Assert.assertNotNull(store.find(trigger.getId())); + Assert.assertTrue(scheduler.fire(trigger.getId())); + Assert.assertTrue(consumed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + + TimeUnit.SECONDS.toNanos(2); + while ((store.find(trigger.getId()) != null + || convergenceAttempts.get() < 2) + && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertNull(store.find(trigger.getId())); + Assert.assertEquals(2, convergenceAttempts.get()); + Assert.assertEquals(1, executions.get()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证死信写入失败后只重放终态协议,不重复执行业务消费者。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldResumeDeadLetterFinalizationWithoutBusinessReplay() + throws Exception { + FailingTerminalStore store = + new FailingTerminalStore( + TerminalOperation.DEAD_LETTER); + Trigger trigger = futureTrigger( + "dead-letter-finalization-retry"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + AtomicInteger executions = new AtomicInteger(); + AtomicInteger convergenceAttempts = + new AtomicInteger(); + CountDownLatch convergenceCompleted = + new CountDownLatch(2); + scheduler.registerConsumer((current, worker) -> { + executions.incrementAndGet(); + throw new NonRetryableTriggerException( + "invalid definition"); + }); + scheduler.registerFailureListener( + (current, failure) -> { + convergenceAttempts.incrementAndGet(); + convergenceCompleted.countDown(); + return true; + }); + + try { + Assert.assertTrue(scheduler.fire( + trigger.getId())); + Assert.assertTrue(store.terminalAttempted.await( + 2, TimeUnit.SECONDS)); + awaitPending(store, trigger.getId()); + Assert.assertTrue(store.find(trigger.getId()) + .isDeadLetterPending()); + + store.fail = false; + Assert.assertTrue(scheduler.fire( + trigger.getId())); + Assert.assertTrue(convergenceCompleted.await( + 2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + + TimeUnit.SECONDS.toNanos(2); + while (store.find(trigger.getId()) != null + && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + + Assert.assertNull(store.find(trigger.getId())); + Assert.assertEquals(1, executions.get()); + Assert.assertEquals( + 2, convergenceAttempts.get()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证旧 owner 失去 claim 后不能提交业务失败终态或移动新 owner 的触发器。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldRejectDeadLetterFinalizationAfterClaimTransfer() + throws Exception { + LostClaimStore store = new LostClaimStore(); + Trigger trigger = futureTrigger( + "claim-transferred"); + store.save(trigger); + TriggerScheduler scheduler = scheduler(store); + AtomicInteger convergenceAttempts = + new AtomicInteger(); + CountDownLatch executed = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> { + executed.countDown(); + throw new NonRetryableTriggerException( + "stale owner"); + }); + scheduler.registerFailureListener( + (current, failure) -> { + convergenceAttempts.incrementAndGet(); + return true; + }); + + try { + Assert.assertTrue(scheduler.fire( + trigger.getId())); + Assert.assertTrue(executed.await( + 2, TimeUnit.SECONDS)); + Assert.assertTrue(store.finalizationRejected.await( + 2, TimeUnit.SECONDS)); + Assert.assertEquals( + 0, convergenceAttempts.get()); + Assert.assertEquals( + 0, store.deadLetterAttempts.get()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证远期触发器只在固定容量内建立精确定时,溢出任务继续由持久仓储保留。 + * + * @throws Exception 反射读取调度状态失败时抛出 + */ + @Test + public void shouldBoundFutureTriggerLocalSchedules() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + TriggerScheduler scheduler = scheduler(store); + try { + for (int index = 0; index < 2000; index++) { + scheduler.schedule(nearFutureTrigger( + "future-" + index, + TimeUnit.SECONDS.toMillis(50))); + } + Field field = TriggerScheduler.class.getDeclaredField( + "scheduledFutures"); + field.setAccessible(true); + @SuppressWarnings("unchecked") + Map localFutures = + (Map) field.get(scheduler); + Assert.assertEquals(1024, localFutures.size()); + Assert.assertEquals(2000, store.findAllPending().size()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证并发登记远期触发器时本地定时表仍严格受容量上限约束。 + * + * @throws Exception 并发调度或反射读取状态失败时抛出 + */ + @Test + public void shouldBoundConcurrentFutureTriggerLocalSchedules() + throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + TriggerScheduler scheduler = scheduler(store); + ExecutorService callers = + Executors.newFixedThreadPool(16); + List> tasks = + new ArrayList<>(); + for (int index = 0; index < 2000; index++) { + int triggerIndex = index; + tasks.add(() -> { + scheduler.schedule(nearFutureTrigger( + "concurrent-future-" + triggerIndex, + TimeUnit.SECONDS.toMillis(50))); + return null; + }); + } + try { + callers.invokeAll(tasks).forEach(result -> { + try { + result.get(); + } catch (Exception error) { + throw new AssertionError( + "concurrent schedule failed", + error); + } + }); + Field field = TriggerScheduler.class.getDeclaredField( + "scheduledFutures"); + field.setAccessible(true); + @SuppressWarnings("unchecked") + Map localFutures = + (Map) field.get(scheduler); + Assert.assertEquals(1024, localFutures.size()); + Assert.assertEquals( + 2000, + store.findAllPending().size()); + } finally { + callers.shutdownNow(); + scheduler.shutdown(); + } + } + + /** + * 验证高并发零延迟触发完成后 Future 与时间索引同时清空。 + * + * @throws Exception 等待任务完成或反射读取索引失败时抛出 + */ + @Test + public void shouldNotLeakImmediateTriggerTimeIndex() + throws Exception { + InMemoryTriggerStore store = + new InMemoryTriggerStore(); + TriggerScheduler scheduler = + scheduler(store); + int triggerCount = 1000; + CountDownLatch consumed = + new CountDownLatch(triggerCount); + scheduler.registerConsumer( + (current, worker) -> + consumed.countDown()); + + try { + for (int index = 0; + index < triggerCount; + index++) { + scheduler.schedule( + dueTrigger( + "immediate-" + index)); + } + Assert.assertTrue( + consumed.await( + 5L, + TimeUnit.SECONDS)); + awaitLocalScheduleIndexEmpty( + scheduler, + "scheduledFutures"); + awaitLocalScheduleIndexEmpty( + scheduler, + "scheduledTriggerTimes"); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证近未来触发器按目标时间执行,不附加完整扫描周期。 + * + * @throws Exception 等待异步触发被中断时抛出 + */ + @Test + public void shouldDispatchFutureTriggerBeforeNextScan() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + TriggerScheduler scheduler = scheduler(store); + CountDownLatch consumed = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> + consumed.countDown()); + Trigger trigger = new Trigger(); + trigger.setId("future-exact"); + trigger.setTriggerAt( + System.currentTimeMillis() + 200L); + + try { + scheduler.schedule(trigger); + Assert.assertTrue(consumed.await( + 700L, TimeUnit.MILLISECONDS)); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证较晚任务占满本地定时容量时,更早到期的任务仍获得精确定时槽位。 + * + * @throws Exception 等待异步触发被中断时抛出 + */ + @Test + public void shouldPreferNearTriggerWhenLocalScheduleIsFull() + throws Exception { + InMemoryTriggerStore store = + new InMemoryTriggerStore(); + TriggerScheduler scheduler = scheduler(store); + CountDownLatch consumed = new CountDownLatch(1); + scheduler.registerConsumer((current, worker) -> { + if ("near-deadline".equals( + current.getId())) { + consumed.countDown(); + } + }); + try { + for (int index = 0; index < 1024; index++) { + scheduler.schedule(nearFutureTrigger( + "later-" + index, + TimeUnit.SECONDS.toMillis(50))); + } + scheduler.schedule(nearFutureTrigger( + "near-deadline", 200L)); + + Assert.assertTrue( + "near trigger should replace a later local timer", + consumed.await( + 700L, + TimeUnit.MILLISECONDS)); + Assert.assertNull( + store.find("near-deadline")); + Assert.assertEquals( + 1024, + store.findAllPending().size()); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证两个调度器并发登记同一稳定入口时,仓储只创建一份待执行触发器。 + * + * @throws Exception 并发调用失败时抛出 + */ + @Test + public void shouldAtomicallyScheduleStableTriggerOnce() + throws Exception { + InMemoryTriggerStore store = + new InMemoryTriggerStore(); + TriggerScheduler first = scheduler(store); + TriggerScheduler second = scheduler(store); + ExecutorService callers = + Executors.newFixedThreadPool(2); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + try { + java.util.concurrent.Future + firstResult = callers.submit(() -> { + ready.countDown(); + start.await(); + return first.scheduleIfAbsent( + futureTrigger( + "stable-entry")); + }); + java.util.concurrent.Future + secondResult = callers.submit(() -> { + ready.countDown(); + start.await(); + return second.scheduleIfAbsent( + futureTrigger( + "stable-entry")); + }); + Assert.assertTrue( + ready.await(1L, TimeUnit.SECONDS)); + start.countDown(); + for (java.util.concurrent.Future result + : Arrays.asList( + firstResult, + secondResult)) { + Assert.assertEquals( + "stable-entry", + result.get().getId()); + } + Assert.assertEquals( + 1, + store.findAllPending().size()); + } finally { + start.countDown(); + callers.shutdownNow(); + first.shutdown(); + second.shutdown(); + } + } + + /** + * 验证工作线程容量暂时耗尽后,持久化触发器可由后续扫描自动恢复执行。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldRescheduleTriggerAfterDispatchCapacityRecovers() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(2); + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>()); + TriggerScheduler scheduler = + new TriggerScheduler(store, scheduledExecutor, worker, 1000L); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondConsumed = new CountDownLatch(1); + scheduler.registerConsumer((current, executor) -> { + if ("capacity-first".equals(current.getId())) { + firstStarted.countDown(); + try { + releaseFirst.await(2, TimeUnit.SECONDS); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("test consumer interrupted", exception); + } + } else if ("capacity-second".equals(current.getId())) { + secondConsumed.countDown(); + } + }); + + try { + scheduler.schedule(dueTrigger("capacity-first")); + Assert.assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + scheduler.schedule(dueTrigger("capacity-second")); + Thread.sleep(100L); + Assert.assertNotNull(store.find("capacity-second")); + + releaseFirst.countDown(); + Assert.assertTrue( + "pending trigger should be rescheduled after capacity recovers", + secondConsumed.await(3, TimeUnit.SECONDS)); + } finally { + releaseFirst.countDown(); + scheduler.shutdown(); + } + } + + /** + * 验证本地调度表被已完成占位填满后,会先清扫并继续接收新任务。 + * + * @throws Exception 反射或异步等待失败时抛出 + */ + @Test + public void shouldPruneCompletedSchedulesBeforeCapacityCheck() throws Exception { + InMemoryTriggerStore store = new InMemoryTriggerStore(); + ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(2); + ExecutorService worker = Executors.newFixedThreadPool(2); + TriggerScheduler scheduler = + new TriggerScheduler(store, scheduledExecutor, worker, 1000L); + CountDownLatch consumed = new CountDownLatch(1); + scheduler.registerConsumer((current, executor) -> consumed.countDown()); + + Field field = TriggerScheduler.class.getDeclaredField("scheduledFutures"); + field.setAccessible(true); + @SuppressWarnings("unchecked") + Map> localFutures = + (Map>) field.get(scheduler); + ScheduledFuture completed = scheduledExecutor.schedule( + () -> { + }, + 0L, + TimeUnit.MILLISECONDS); + completed.get(1, TimeUnit.SECONDS); + for (int index = 0; index < 1024; index++) { + localFutures.put("completed-" + index, completed); + } + + try { + scheduler.schedule(dueTrigger("after-completed-capacity")); + Assert.assertTrue( + "completed local schedules should not block new due triggers", + consumed.await(2, TimeUnit.SECONDS)); + Assert.assertTrue(localFutures.size() < 1024); + } finally { + scheduler.shutdown(); + } + } + + /** + * 验证确认成功状态写入异常时仍归还调度容量许可。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldReturnDispatchPermitWhenAcknowledgeFails() throws Exception { + assertTerminalStoreFailureDoesNotLeakPermit(TerminalOperation.ACKNOWLEDGE); + } + + /** + * 验证失败重投状态写入异常时仍归还调度容量许可。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldReturnDispatchPermitWhenReleaseFails() throws Exception { + assertTerminalStoreFailureDoesNotLeakPermit(TerminalOperation.RELEASE); + } + + /** + * 验证死信状态写入异常时仍归还调度容量许可。 + * + * @throws Exception 等待异步执行被中断时抛出 + */ + @Test + public void shouldReturnDispatchPermitWhenDeadLetterFails() throws Exception { + assertTerminalStoreFailureDoesNotLeakPermit(TerminalOperation.DEAD_LETTER); + } + + /** + * 执行终态仓储异常后的容量许可回归场景。 + * + * @param operation 需要模拟失败的终态操作 + * @throws Exception 等待异步执行被中断时抛出 + */ + private void assertTerminalStoreFailureDoesNotLeakPermit( + TerminalOperation operation) throws Exception { + FailingTerminalStore store = new FailingTerminalStore(operation); + ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(2); + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>()); + TriggerScheduler scheduler = + new TriggerScheduler(store, scheduledExecutor, worker, 1000L); + CountDownLatch secondConsumed = new CountDownLatch(1); + scheduler.registerConsumer((current, executor) -> { + if ("second".equals(current.getId())) { + secondConsumed.countDown(); + return; + } + if (operation == TerminalOperation.DEAD_LETTER) { + throw new NonRetryableTriggerException("expected dead-letter failure"); + } + if (operation == TerminalOperation.RELEASE) { + throw new IllegalStateException("expected release failure"); + } + }); + + try { + Trigger first = futureTrigger("first"); + store.save(first); + Assert.assertTrue(scheduler.fire(first.getId())); + Assert.assertTrue(store.terminalAttempted.await(2, TimeUnit.SECONDS)); + + store.fail = false; + Trigger second = futureTrigger("second"); + store.save(second); + boolean accepted = false; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (!accepted && System.nanoTime() < deadline) { + accepted = scheduler.fire(second.getId()); + if (!accepted) { + Thread.sleep(10L); + } + } + Assert.assertTrue("dispatch permit should be returned", accepted); + Assert.assertTrue(secondConsumed.await(2, TimeUnit.SECONDS)); + } finally { + scheduler.shutdown(); + } + } + + /** + * 创建测试调度器。 + * + * @param store 触发器仓储 + * @return 测试调度器 + */ + private TriggerScheduler scheduler(TriggerStore store) { + ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(2); + ExecutorService worker = Executors.newFixedThreadPool(2); + return new TriggerScheduler(store, scheduledExecutor, worker, 1000L); + } + + /** + * 等待指定本地调度索引清空。 + * + * @param scheduler 调度器 + * @param fieldName 索引字段名 + * @throws Exception 反射读取失败或等待超时时抛出 + */ + private void awaitLocalScheduleIndexEmpty( + TriggerScheduler scheduler, + String fieldName) throws Exception { + Field field = + TriggerScheduler.class.getDeclaredField( + fieldName); + field.setAccessible(true); + @SuppressWarnings("unchecked") + Map index = + (Map) field.get(scheduler); + long deadline = + System.nanoTime() + + TimeUnit.SECONDS.toNanos(2L); + while (!index.isEmpty() + && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertTrue( + fieldName + " should be empty", + index.isEmpty()); + } + + /** + * 创建远期触发器,避免定时任务干扰主动认领测试。 + * + * @param id 触发器 ID + * @return 测试触发器 + */ + private Trigger futureTrigger(String id) { + return nearFutureTrigger( + id, + TimeUnit.MINUTES.toMillis(5)); + } + + /** + * 创建指定延迟的测试触发器。 + * + * @param id 触发器 ID + * @param delayMillis 延迟毫秒数 + * @return 测试触发器 + */ + private Trigger nearFutureTrigger( + String id, + long delayMillis) { + Trigger trigger = new Trigger(); + trigger.setId(id); + trigger.setTriggerAt( + System.currentTimeMillis() + + delayMillis); + return trigger; + } + + /** + * 创建立即到期的测试触发器。 + * + * @param id 触发器 ID + * @return 测试触发器 + */ + private Trigger dueTrigger(String id) { + Trigger trigger = new Trigger(); + trigger.setId(id); + trigger.setTriggerAt(System.currentTimeMillis()); + return trigger; + } + + /** + * 等待失败触发器重新进入待执行状态。 + * + * @param store 触发器仓储 + * @param triggerId 触发器 ID + * @throws InterruptedException 等待被中断时抛出 + */ + private void awaitPending(InMemoryTriggerStore store, String triggerId) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (store.find(triggerId) == null && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertNotNull(store.find(triggerId)); + } + + /** + * 需要模拟失败的触发器终态操作。 + */ + private enum TerminalOperation { + ACKNOWLEDGE, + RELEASE, + DEAD_LETTER + } + + /** + * 在指定终态操作上抛出异常的进程内触发器仓储。 + */ + private static final class FailingTerminalStore extends InMemoryTriggerStore { + + private final TerminalOperation operation; + private final CountDownLatch terminalAttempted = new CountDownLatch(1); + private volatile boolean fail = true; + + /** + * 创建失败仓储。 + * + * @param operation 需要失败的终态操作 + */ + private FailingTerminalStore(TerminalOperation operation) { + this.operation = operation; + } + + /** + * {@inheritDoc} + */ + @Override + public void acknowledge(Trigger trigger) { + failIfNeeded(TerminalOperation.ACKNOWLEDGE); + super.acknowledge(trigger); + } + + /** + * {@inheritDoc} + */ + @Override + public void release(Trigger trigger) { + failIfNeeded(TerminalOperation.RELEASE); + super.release(trigger); + } + + /** + * {@inheritDoc} + */ + @Override + public void deadLetter(Trigger trigger, String reason) { + failIfNeeded(TerminalOperation.DEAD_LETTER); + super.deadLetter(trigger, reason); + } + + /** + * 在当前测试操作上抛出预期异常。 + * + * @param current 当前终态操作 + */ + private void failIfNeeded(TerminalOperation current) { + if (fail && operation == current) { + terminalAttempted.countDown(); + throw new IllegalStateException("expected terminal store failure: " + current); + } + } + } + + /** + * 模拟 claim 已转移给其他 owner 的触发器仓储。 + */ + private static final class LostClaimStore + extends InMemoryTriggerStore { + + private final CountDownLatch finalizationRejected = + new CountDownLatch(1); + private final AtomicInteger deadLetterAttempts = + new AtomicInteger(); + + /** + * {@inheritDoc} + */ + @Override + public boolean renewClaim( + Trigger trigger, long leaseMillis) { + finalizationRejected.countDown(); + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public void deadLetter( + Trigger trigger, String reason) { + deadLetterAttempts.incrementAndGet(); + } + } +} From 12491b372484f8387596d4038359fbc4cea31696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 29 Jul 2026 01:03:42 +0800 Subject: [PATCH 13/33] =?UTF-8?q?feat:=20=E5=A2=9E=E5=BC=BA=E5=B8=B8?= =?UTF-8?q?=E8=A7=84=E6=96=87=E6=A1=A3=E8=BD=BB=E9=87=8F=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持 PDF、Office、表格、TXT 与 Markdown 的结构化轻量读取 - 增加结构上限、取消信号、稳定定位与读取错误分类 - 使用事件流读取表格并补充核心读取测试 --- .../core/file2text/DocumentReadErrorCode.java | 37 +++ .../core/file2text/DocumentReadException.java | 56 ++++ .../core/file2text/DocumentReadSupport.java | 107 ++++++++ .../core/file2text/DocumentTextSegment.java | 80 ++++++ .../core/file2text/File2TextService.java | 80 +++++- .../LightweightDocumentReadRequest.java | 100 +++++++ .../LightweightDocumentReadResult.java | 91 +++++++ .../extractor/ExtractorRegistry.java | 3 + .../file2text/extractor/FileExtractor.java | 37 ++- .../extractor/impl/DocExtractor.java | 52 +++- .../extractor/impl/DocxExtractor.java | 130 +++++++-- .../extractor/impl/PdfTextExtractor.java | 55 +++- .../extractor/impl/PlainTextExtractor.java | 181 ++++++++++++- .../extractor/impl/PptExtractor.java | 122 +++++++++ .../extractor/impl/PptxExtractor.java | 51 +++- .../impl/SpreadsheetReadSupport.java | 76 ++++++ .../extractor/impl/XlsExtractor.java | 256 ++++++++++++++++++ .../extractor/impl/XlsxExtractor.java | 228 ++++++++++++++++ .../File2TextServiceLightweightReadTest.java | 162 +++++++++++ 19 files changed, 1832 insertions(+), 72 deletions(-) create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadErrorCode.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadException.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadSupport.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentTextSegment.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadRequest.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadResult.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptExtractor.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/SpreadsheetReadSupport.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsExtractor.java create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsxExtractor.java create mode 100644 easy-agents-core/src/test/java/com/easyagents/core/file2text/File2TextServiceLightweightReadTest.java diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadErrorCode.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadErrorCode.java new file mode 100644 index 0000000..8632c51 --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadErrorCode.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.core.file2text; + +/** + * 轻量文档读取错误码。 + */ +public enum DocumentReadErrorCode { + + /** 不支持的文档类型。 */ + UNSUPPORTED_DOCUMENT_TYPE, + /** 文档结构超过安全上限。 */ + DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + /** 文档已加密。 */ + DOCUMENT_ENCRYPTED, + /** 文档损坏或容器不合法。 */ + DOCUMENT_CORRUPTED, + /** 文档中没有可读取文字。 */ + DOCUMENT_NO_READABLE_TEXT, + /** 文档读取已取消。 */ + DOCUMENT_READ_CANCELLED, + /** 未分类的文档读取失败。 */ + DOCUMENT_READ_FAILED +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadException.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadException.java new file mode 100644 index 0000000..c63b78f --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadException.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.core.file2text; + +/** + * 轻量文档读取异常。 + */ +public class DocumentReadException extends RuntimeException { + + private final DocumentReadErrorCode errorCode; + + /** + * 创建文档读取异常。 + * + * @param errorCode 错误码 + * @param message 错误消息 + */ + public DocumentReadException(DocumentReadErrorCode errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + /** + * 创建带原因的文档读取异常。 + * + * @param errorCode 错误码 + * @param message 错误消息 + * @param cause 原始异常 + */ + public DocumentReadException(DocumentReadErrorCode errorCode, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + } + + /** + * 获取错误码。 + * + * @return 错误码 + */ + public DocumentReadErrorCode getErrorCode() { + return errorCode; + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadSupport.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadSupport.java new file mode 100644 index 0000000..dbedeba --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentReadSupport.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text; + +import com.easyagents.core.file2text.source.DocumentSource; + +import java.util.List; + +/** + * 结构化文档读取结果构造工具。 + */ +public final class DocumentReadSupport { + + private DocumentReadSupport() { + } + + /** + * 创建带定位的文本片段。 + * + * @param segmentId 片段 ID + * @param text 文本 + * @param locatorType 定位类型 + * @param locatorLabel 定位标签 + * @param startIndex 起始字符下标 + * @param headingPath 标题路径 + * @return 文本片段 + */ + public static DocumentTextSegment segment(String segmentId, + String text, + String locatorType, + String locatorLabel, + int startIndex, + List headingPath) { + String safeText = text == null ? "" : text.trim(); + DocumentTextSegment segment = new DocumentTextSegment(); + segment.setSegmentId(segmentId); + segment.setText(safeText); + segment.setLocatorType(locatorType); + segment.setLocatorLabel(locatorLabel); + segment.setStartIndex(startIndex); + segment.setEndIndex(startIndex + safeText.length()); + segment.setHeadingPath(headingPath); + segment.setTokenEstimate(estimateTokens(safeText)); + return segment; + } + + /** + * 汇总结构化读取结果。 + * + * @param source 文档来源 + * @param segments 片段 + * @param request 读取请求 + * @return 读取结果 + * @throws DocumentReadException 结果为空或超过边界 + */ + public static LightweightDocumentReadResult result(DocumentSource source, + List segments, + LightweightDocumentReadRequest request) + throws DocumentReadException { + List nonEmpty = segments.stream() + .filter(item -> item.getText() != null && !item.getText().isBlank()) + .toList(); + if (nonEmpty.isEmpty()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_NO_READABLE_TEXT, + "No readable text detected"); + } + long charCount = nonEmpty.stream().mapToLong(item -> item.getText().length()).sum() + + Math.max(0, nonEmpty.size() - 1L); + if (charCount > request.getMaxExpandedChars() || charCount > Integer.MAX_VALUE) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + "Expanded document text exceeds the configured limit"); + } + LightweightDocumentReadResult result = new LightweightDocumentReadResult(); + result.setFileName(source.getFileName()); + result.setMimeType(source.getMimeType()); + result.setSegments(nonEmpty); + result.setCharCount((int) charCount); + result.setTokenEstimate(nonEmpty.stream().mapToInt(DocumentTextSegment::getTokenEstimate).sum()); + return result; + } + + /** + * 使用偏保守的字符规则估算 Token 数。 + * + * @param text 文本 + * @return Token 估算 + */ + public static int estimateTokens(String text) { + if (text == null || text.isEmpty()) { + return 0; + } + double tokens = 0; + for (int offset = 0; offset < text.length();) { + int codePoint = text.codePointAt(offset); + Character.UnicodeScript script = Character.UnicodeScript.of(codePoint); + tokens += switch (script) { + case HAN, HANGUL, HIRAGANA, KATAKANA -> 1.0d; + default -> Character.isWhitespace(codePoint) ? 0.1d : 0.25d; + }; + offset += Character.charCount(codePoint); + } + return Math.max(1, (int) Math.ceil(tokens)); + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentTextSegment.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentTextSegment.java new file mode 100644 index 0000000..774be37 --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/DocumentTextSegment.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text; + +import java.util.ArrayList; +import java.util.List; + +/** + * 带稳定定位信息的文档文本片段。 + */ +public class DocumentTextSegment { + + private String segmentId; + private String text; + private String locatorType; + private String locatorLabel; + private int startIndex; + private int endIndex; + private List headingPath = new ArrayList<>(); + private int tokenEstimate; + + /** @return 片段 ID */ + public String getSegmentId() { return segmentId; } + + /** @param segmentId 片段 ID */ + public void setSegmentId(String segmentId) { this.segmentId = segmentId; } + + /** @return 片段文本 */ + public String getText() { return text; } + + /** @param text 片段文本 */ + public void setText(String text) { this.text = text; } + + /** @return 定位类型 */ + public String getLocatorType() { return locatorType; } + + /** @param locatorType 定位类型 */ + public void setLocatorType(String locatorType) { this.locatorType = locatorType; } + + /** @return 可读定位标签 */ + public String getLocatorLabel() { return locatorLabel; } + + /** @param locatorLabel 可读定位标签 */ + public void setLocatorLabel(String locatorLabel) { this.locatorLabel = locatorLabel; } + + /** @return 全文起始字符下标 */ + public int getStartIndex() { return startIndex; } + + /** @param startIndex 全文起始字符下标 */ + public void setStartIndex(int startIndex) { this.startIndex = startIndex; } + + /** @return 全文结束字符下标 */ + public int getEndIndex() { return endIndex; } + + /** @param endIndex 全文结束字符下标 */ + public void setEndIndex(int endIndex) { this.endIndex = endIndex; } + + /** @return 标题路径 */ + public List getHeadingPath() { return headingPath; } + + /** @param headingPath 标题路径 */ + public void setHeadingPath(List headingPath) { + this.headingPath = headingPath == null ? new ArrayList<>() : new ArrayList<>(headingPath); + } + + /** @return Token 估算 */ + public int getTokenEstimate() { return tokenEstimate; } + + /** @param tokenEstimate Token 估算 */ + public void setTokenEstimate(int tokenEstimate) { this.tokenEstimate = tokenEstimate; } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/File2TextService.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/File2TextService.java index 281a1ae..e65a2bb 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/File2TextService.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/File2TextService.java @@ -22,9 +22,13 @@ import com.easyagents.core.file2text.source.*; import java.io.File; import java.io.InputStream; +import java.io.IOException; import java.util.List; import java.util.stream.Collectors; +/** + * 文档轻量读取服务。 + */ public class File2TextService { private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(File2TextService.class); private final ExtractorRegistry registry; @@ -75,6 +79,44 @@ public class File2TextService { * @throws IllegalArgumentException 输入源为空 */ public String extractTextFromSource(DocumentSource source) { + return readFromSource(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 从文件读取结构化文档内容。 + * + * @param file 文档文件 + * @return 结构化结果 + */ + public LightweightDocumentReadResult readFromFile(File file) { + return readFromSource(new LightweightDocumentReadRequest(new FileDocumentSource(file))); + } + + /** + * 从输入流读取结构化文档内容。 + * + * @param inputStream 文档输入流 + * @param fileName 文件名 + * @param mimeType MIME 类型 + * @return 结构化结果 + */ + public LightweightDocumentReadResult readFromStream(InputStream inputStream, String fileName, String mimeType) { + return readFromSource(new LightweightDocumentReadRequest( + new ByteStreamDocumentSource(inputStream, fileName, mimeType))); + } + + /** + * 按请求读取结构化文档内容。 + * + * @param request 读取请求 + * @return 结构化结果 + * @throws DocumentReadException 不支持、空文本或读取失败 + */ + public LightweightDocumentReadResult readFromSource(LightweightDocumentReadRequest request) { + if (request == null || request.getSource() == null) { + throw new IllegalArgumentException("Document read request cannot be null"); + } + DocumentSource source = request.getSource(); if (source == null) { throw new IllegalArgumentException("DocumentSource cannot be null"); } @@ -83,8 +125,8 @@ public class File2TextService { // 获取可用的 Extractor(按优先级排序) List candidates = registry.findExtractors(source); if (candidates.isEmpty()) { - log.warn("No extractor supports this document: " + safeFileName(source)); - return null; + throw new DocumentReadException(DocumentReadErrorCode.UNSUPPORTED_DOCUMENT_TYPE, + "Unsupported document type: " + safeFileName(source)); } // 日志:输出候选 Extractor @@ -93,29 +135,41 @@ public class File2TextService { .map(e -> e.getClass().getSimpleName()) .collect(Collectors.joining(", "))); - + DocumentReadException lastFailure = null; for (FileExtractor extractor : candidates) { try { log.debug("Trying {} on {}", extractor.getClass().getSimpleName(), safeFileName(source)); - - String text = extractor.extractText(source); - if (text != null && !text.trim().isEmpty()) { + LightweightDocumentReadResult result = extractor.read(request); + if (result != null && !result.getSegments().isEmpty()) { log.debug("Success with {}: extracted {} chars", - extractor.getClass().getSimpleName(), text.length()); - return text; - } else { - log.debug("Extractor {} returned null", extractor.getClass().getSimpleName()); + extractor.getClass().getSimpleName(), result.getCharCount()); + return result; } - } catch (Exception e) { + } catch (DocumentReadException e) { + lastFailure = e; + log.warn("Extractor {} rejected {} with {}: {}", + extractor.getClass().getSimpleName(), safeFileName(source), + e.getErrorCode(), e.getMessage()); + } catch (IOException e) { + lastFailure = new DocumentReadException(DocumentReadErrorCode.DOCUMENT_READ_FAILED, + "Failed to read document: " + safeFileName(source), e); log.warn("Extractor {} failed on {}: {}", extractor.getClass().getSimpleName(), safeFileName(source), e.toString()); + } catch (RuntimeException e) { + lastFailure = new DocumentReadException(DocumentReadErrorCode.DOCUMENT_READ_FAILED, + "Failed to read document: " + safeFileName(source), e); + log.warn("Extractor {} failed on {}: {}", + extractor.getClass().getSimpleName(), safeFileName(source), e.toString()); } } - log.warn(String.format("All %d extractors failed for: %s", candidates.size(), safeFileName(source))); - return null; + if (lastFailure != null) { + throw lastFailure; + } + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_NO_READABLE_TEXT, + "No readable text detected: " + safeFileName(source)); } finally { source.cleanup(); } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadRequest.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadRequest.java new file mode 100644 index 0000000..1b8d835 --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadRequest.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text; + +import com.easyagents.core.file2text.source.DocumentSource; + +import java.util.Objects; +import java.util.function.BooleanSupplier; + +/** + * 轻量文档读取请求及结构安全边界。 + */ +public class LightweightDocumentReadRequest { + + private final DocumentSource source; + private int maxPdfPages = 200; + private int maxSlides = 200; + private int maxSheets = 20; + private int maxNonEmptyCells = 50_000; + private long maxExpandedChars = 150L * 1024L * 1024L; + private BooleanSupplier cancelled = () -> false; + + /** + * 创建读取请求。 + * + * @param source 文档来源 + */ + public LightweightDocumentReadRequest(DocumentSource source) { + this.source = Objects.requireNonNull(source, "DocumentSource cannot be null"); + } + + /** @return 文档来源 */ + public DocumentSource getSource() { return source; } + + /** @return 最大 PDF 页数 */ + public int getMaxPdfPages() { return maxPdfPages; } + + /** @param maxPdfPages 最大 PDF 页数 */ + public void setMaxPdfPages(int maxPdfPages) { this.maxPdfPages = positive(maxPdfPages, "maxPdfPages"); } + + /** @return 最大幻灯片数 */ + public int getMaxSlides() { return maxSlides; } + + /** @param maxSlides 最大幻灯片数 */ + public void setMaxSlides(int maxSlides) { this.maxSlides = positive(maxSlides, "maxSlides"); } + + /** @return 最大工作表数 */ + public int getMaxSheets() { return maxSheets; } + + /** @param maxSheets 最大工作表数 */ + public void setMaxSheets(int maxSheets) { this.maxSheets = positive(maxSheets, "maxSheets"); } + + /** @return 最大非空单元格数 */ + public int getMaxNonEmptyCells() { return maxNonEmptyCells; } + + /** @param maxNonEmptyCells 最大非空单元格数 */ + public void setMaxNonEmptyCells(int maxNonEmptyCells) { + this.maxNonEmptyCells = positive(maxNonEmptyCells, "maxNonEmptyCells"); + } + + /** @return 最大展开字符数 */ + public long getMaxExpandedChars() { return maxExpandedChars; } + + /** @param maxExpandedChars 最大展开字符数 */ + public void setMaxExpandedChars(long maxExpandedChars) { + if (maxExpandedChars <= 0) { + throw new IllegalArgumentException("maxExpandedChars must be positive"); + } + this.maxExpandedChars = maxExpandedChars; + } + + /** @return 取消检查器 */ + public BooleanSupplier getCancelled() { return cancelled; } + + /** @param cancelled 取消检查器 */ + public void setCancelled(BooleanSupplier cancelled) { + this.cancelled = cancelled == null ? () -> false : cancelled; + } + + /** + * 检查当前读取是否已取消。 + * + * @throws DocumentReadException 已取消时抛出 + */ + public void checkCancelled() throws DocumentReadException { + if (cancelled.getAsBoolean() || Thread.currentThread().isInterrupted()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_READ_CANCELLED, "Document read cancelled"); + } + } + + private int positive(int value, String name) { + if (value <= 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return value; + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadResult.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadResult.java new file mode 100644 index 0000000..b1494cd --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/LightweightDocumentReadResult.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text; + +import java.util.ArrayList; +import java.util.List; + +/** + * 轻量文档结构化读取结果。 + */ +public class LightweightDocumentReadResult { + + /** 当前读取器版本。 */ + public static final String READER_VERSION = "v1"; + /** 当前读取策略版本。 */ + public static final String READ_POLICY_VERSION = "v1"; + + private String fileName; + private String mimeType; + private String readerVersion = READER_VERSION; + private String readPolicyVersion = READ_POLICY_VERSION; + private int charCount; + private int tokenEstimate; + private List segments = new ArrayList<>(); + + /** @return 文件名 */ + public String getFileName() { return fileName; } + + /** @param fileName 文件名 */ + public void setFileName(String fileName) { this.fileName = fileName; } + + /** @return MIME 类型 */ + public String getMimeType() { return mimeType; } + + /** @param mimeType MIME 类型 */ + public void setMimeType(String mimeType) { this.mimeType = mimeType; } + + /** @return 读取器版本 */ + public String getReaderVersion() { return readerVersion; } + + /** @param readerVersion 读取器版本 */ + public void setReaderVersion(String readerVersion) { this.readerVersion = readerVersion; } + + /** @return 读取策略版本 */ + public String getReadPolicyVersion() { return readPolicyVersion; } + + /** @param readPolicyVersion 读取策略版本 */ + public void setReadPolicyVersion(String readPolicyVersion) { this.readPolicyVersion = readPolicyVersion; } + + /** @return 字符数 */ + public int getCharCount() { return charCount; } + + /** @param charCount 字符数 */ + public void setCharCount(int charCount) { this.charCount = charCount; } + + /** @return Token 估算 */ + public int getTokenEstimate() { return tokenEstimate; } + + /** @param tokenEstimate Token 估算 */ + public void setTokenEstimate(int tokenEstimate) { this.tokenEstimate = tokenEstimate; } + + /** @return 文本片段 */ + public List getSegments() { return segments; } + + /** @param segments 文本片段 */ + public void setSegments(List segments) { + this.segments = segments == null ? new ArrayList<>() : new ArrayList<>(segments); + } + + /** + * 按片段顺序拼接兼容纯文本。 + * + * @return 拼接后的文本 + */ + public String getText() { + StringBuilder text = new StringBuilder(Math.max(0, charCount)); + for (DocumentTextSegment segment : segments) { + if (segment.getText() == null || segment.getText().isBlank()) { + continue; + } + if (!text.isEmpty()) { + text.append('\n'); + } + text.append(segment.getText()); + } + return text.toString(); + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/ExtractorRegistry.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/ExtractorRegistry.java index a2a905a..96d5694 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/ExtractorRegistry.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/ExtractorRegistry.java @@ -35,7 +35,10 @@ public class ExtractorRegistry { register(new PdfTextExtractor()); register(new DocxExtractor()); register(new DocExtractor()); + register(new PptExtractor()); register(new PptxExtractor()); + register(new XlsExtractor()); + register(new XlsxExtractor()); register(new HtmlExtractor()); register(new PlainTextExtractor()); } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/FileExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/FileExtractor.java index 62d50ca..44893fd 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/FileExtractor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/FileExtractor.java @@ -15,24 +15,59 @@ */ package com.easyagents.core.file2text.extractor; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; import com.easyagents.core.file2text.source.DocumentSource; import java.io.IOException; import java.util.Comparator; +import java.util.List; +/** + * 文档文本提取器。 + */ public interface FileExtractor { Comparator ORDER_COMPARATOR = Comparator.comparingInt(FileExtractor::getOrder); /** - * 判断该 Extractor 是否支持处理此文档 + * 判断该 Extractor 是否支持处理此文档。 + * + * @param source 文档来源 + * @return 是否支持 */ boolean supports(DocumentSource source); + /** + * 提取兼容纯文本。 + * + * @param source 文档来源 + * @return 提取文本 + * @throws IOException 文档读取失败 + */ String extractText(DocumentSource source) throws IOException; + /** + * 提取结构化文档片段。 + * + * @param request 读取请求 + * @return 结构化结果 + * @throws IOException 文档读取失败 + */ + default LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + String text = extractText(request.getSource()); + return DocumentReadSupport.result(request.getSource(), + List.of(DocumentReadSupport.segment("segment-1", text, "DOCUMENT", "全文", 0, List.of())), + request); + } + /** + * 获取读取器优先级。 + * + * @return 越小越优先 + */ default int getOrder() { return 100; } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocExtractor.java index 46f8d22..f1d0aa1 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocExtractor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocExtractor.java @@ -15,6 +15,12 @@ */ package com.easyagents.core.file2text.extractor.impl; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; import com.easyagents.core.file2text.extractor.FileExtractor; import com.easyagents.core.file2text.source.DocumentSource; import org.apache.poi.hwpf.HWPFDocument; @@ -24,7 +30,9 @@ import org.apache.poi.poifs.filesystem.POIFSFileSystem; import java.io.IOException; import java.io.InputStream; import java.util.Collections; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -68,25 +76,47 @@ public class DocExtractor implements FileExtractor { @Override public String extractText(DocumentSource source) throws IOException { + return read(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 按段落读取 Word 97-2003 文档。 + * + * @param request 读取请求 + * @return 段落结构化结果 + * @throws IOException 文档 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); try (InputStream is = source.openStream(); POIFSFileSystem fs = new POIFSFileSystem(is); HWPFDocument doc = new HWPFDocument(fs)) { - WordExtractor extractor = new WordExtractor(doc); - String[] paragraphs = extractor.getParagraphText(); - - StringBuilder text = new StringBuilder(); - for (String para : paragraphs) { + try (WordExtractor extractor = new WordExtractor(doc)) { + String[] paragraphs = extractor.getParagraphText(); + List segments = new ArrayList<>(); + int offset = 0; + for (int index = 0; index < paragraphs.length; index++) { + request.checkCancelled(); + String para = paragraphs[index]; // 清理控制字符 - String clean = para.replaceAll("[\\r\\001]+", "").trim(); - if (!clean.isEmpty()) { - text.append(clean).append("\n"); + String clean = para.replaceAll("[\\r\\001]+", "").trim(); + if (!clean.isEmpty()) { + DocumentTextSegment segment = DocumentReadSupport.segment( + "paragraph-" + (index + 1), clean, "PARAGRAPH", + "第 " + (index + 1) + " 段", offset, List.of()); + segments.add(segment); + offset = segment.getEndIndex() + 1; + } } + return DocumentReadSupport.result(source, segments, request); } - - return text.toString().trim(); + } catch (DocumentReadException e) { + throw e; } catch (Exception e) { - throw new IOException("Failed to extract .doc file: " + e.getMessage(), e); + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract .doc file", e); } } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocxExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocxExtractor.java index 3f8dc4d..1e74973 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocxExtractor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/DocxExtractor.java @@ -16,12 +16,19 @@ package com.easyagents.core.file2text.extractor.impl; +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; import com.easyagents.core.file2text.extractor.FileExtractor; import com.easyagents.core.file2text.source.DocumentSource; import org.apache.poi.xwpf.usermodel.*; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -79,37 +86,66 @@ public class DocxExtractor implements FileExtractor { @Override public String extractText(DocumentSource source) throws IOException { - StringBuilder text = new StringBuilder(); + return read(new LightweightDocumentReadRequest(source)).getText(); + } + /** + * 按正文顺序读取 DOCX 段落和表格。 + * + * @param request 读取请求 + * @return 结构化读取结果 + * @throws IOException 文档 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); try (InputStream is = source.openStream(); XWPFDocument document = new XWPFDocument(is)) { - - // 提取段落 - for (XWPFParagraph paragraph : document.getParagraphs()) { - String paraText = getParagraphText(paragraph); - if (paraText != null && !paraText.trim().isEmpty()) { - text.append(paraText).append("\n"); + List segments = new ArrayList<>(); + List headingPath = new ArrayList<>(); + int paragraphIndex = 0; + int tableIndex = 0; + int offset = 0; + for (IBodyElement element : document.getBodyElements()) { + request.checkCancelled(); + if (element instanceof XWPFParagraph paragraph) { + paragraphIndex++; + String text = getParagraphText(paragraph); + if (text == null || text.isBlank()) { + continue; + } + int headingLevel = headingLevel(paragraph); + if (headingLevel > 0) { + while (headingPath.size() >= headingLevel) { + headingPath.remove(headingPath.size() - 1); + } + headingPath.add(text.trim()); + } + DocumentTextSegment segment = DocumentReadSupport.segment( + "paragraph-" + paragraphIndex, text, "PARAGRAPH", + "第 " + paragraphIndex + " 段", offset, headingPath); + segments.add(segment); + offset = segment.getEndIndex() + 1; + } else if (element instanceof XWPFTable table) { + tableIndex++; + String tableText = getTableText(table); + if (tableText.isBlank()) { + continue; + } + DocumentTextSegment segment = DocumentReadSupport.segment( + "table-" + tableIndex, tableText, "TABLE", + "第 " + tableIndex + " 个表格", offset, headingPath); + segments.add(segment); + offset = segment.getEndIndex() + 1; } } - - // 提取表格 - for (XWPFTable table : document.getTables()) { - text.append("\n[Table Start]\n"); - for (XWPFTableRow row : table.getRows()) { - List cellTexts = row.getTableCells().stream() - .map(this::getCellText) - .map(String::trim) - .collect(Collectors.toList()); - text.append(cellTexts).append("\n"); - } - text.append("[Table End]\n\n"); - } - + return DocumentReadSupport.result(source, segments, request); + } catch (DocumentReadException e) { + throw e; } catch (Exception e) { - throw new IOException("Failed to extract DOCX: " + e.getMessage(), e); + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract DOCX", e); } - - return text.toString().trim(); } private String getParagraphText(XWPFParagraph paragraph) { @@ -138,6 +174,52 @@ public class DocxExtractor implements FileExtractor { return text.toString().trim(); } + /** + * 获取表格的显示文本。 + * + * @param table 表格 + * @return 表格文本 + */ + private String getTableText(XWPFTable table) { + StringBuilder text = new StringBuilder(); + for (XWPFTableRow row : table.getRows()) { + List cellTexts = row.getTableCells().stream() + .map(this::getCellText) + .map(String::trim) + .collect(Collectors.toList()); + if (cellTexts.stream().anyMatch(item -> !item.isEmpty())) { + text.append(String.join(" | ", cellTexts)).append('\n'); + } + } + return text.toString().trim(); + } + + /** + * 解析常见 Word 标题样式层级。 + * + * @param paragraph 段落 + * @return 标题层级,非标题返回 0 + */ + private int headingLevel(XWPFParagraph paragraph) { + String style = paragraph.getStyle(); + if (style == null) { + return 0; + } + String normalized = style.replaceAll("\\s+", "").toLowerCase(); + if (!normalized.startsWith("heading") && !normalized.startsWith("标题")) { + return 0; + } + String digits = normalized.replaceAll("\\D+", ""); + if (digits.isEmpty()) { + return 1; + } + try { + return Math.max(1, Math.min(9, Integer.parseInt(digits))); + } catch (NumberFormatException ignored) { + return 1; + } + } + @Override public int getOrder() { return 10; diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PdfTextExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PdfTextExtractor.java index 8c28c08..61acbf4 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PdfTextExtractor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PdfTextExtractor.java @@ -15,15 +15,24 @@ */ package com.easyagents.core.file2text.extractor.impl; +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; import com.easyagents.core.file2text.extractor.FileExtractor; import com.easyagents.core.file2text.source.DocumentSource; import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException; import org.apache.pdfbox.text.PDFTextStripper; import java.io.IOException; import java.io.InputStream; import java.util.Collections; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -64,12 +73,54 @@ public class PdfTextExtractor implements FileExtractor { @Override public String extractText(DocumentSource source) throws IOException { + return read(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 按页读取 PDF 文本层。 + * + * @param request 读取请求 + * @return 按页结构化结果 + * @throws IOException PDF I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); try (InputStream is = source.openStream(); PDDocument doc = PDDocument.load(is)) { + if (doc.isEncrypted()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_ENCRYPTED, + "Encrypted PDF is not supported"); + } + int pages = doc.getNumberOfPages(); + if (pages > request.getMaxPdfPages()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + "PDF page count exceeds " + request.getMaxPdfPages()); + } PDFTextStripper stripper = new PDFTextStripper(); - return stripper.getText(doc).trim(); + List segments = new ArrayList<>(); + int offset = 0; + for (int page = 1; page <= pages; page++) { + request.checkCancelled(); + stripper.setStartPage(page); + stripper.setEndPage(page); + String text = stripper.getText(doc).trim(); + if (!text.isEmpty()) { + DocumentTextSegment segment = DocumentReadSupport.segment( + "page-" + page, text, "PAGE", "第 " + page + " 页", offset, List.of()); + segments.add(segment); + offset = segment.getEndIndex() + 1; + } + } + return DocumentReadSupport.result(source, segments, request); + } catch (InvalidPasswordException e) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_ENCRYPTED, + "Encrypted PDF is not supported", e); + } catch (DocumentReadException e) { + throw e; } catch (Exception e) { - throw new IOException("Failed to extract PDF text: " + e.getMessage(), e); + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract PDF text", e); } } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PlainTextExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PlainTextExtractor.java index 2029f8a..10add06 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PlainTextExtractor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PlainTextExtractor.java @@ -16,6 +16,12 @@ package com.easyagents.core.file2text.extractor.impl; +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; import com.easyagents.core.file2text.extractor.FileExtractor; import com.easyagents.core.file2text.source.DocumentSource; @@ -23,9 +29,17 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * 纯文本文件提取器(支持 UTF-8、GBK、GB2312 编码自动检测) @@ -33,6 +47,9 @@ import java.util.Set; */ public class PlainTextExtractor implements FileExtractor { + private static final int LINES_PER_SEGMENT = 40; + private static final Charset GB18030 = Charset.forName("GB18030"); + private static final Pattern MARKDOWN_HEADING = Pattern.compile("^(#{1,6})\\s+(.+?)\\s*$"); private static final Set SUPPORTED_MIME_TYPES; private static final Set SUPPORTED_EXTENSIONS; @@ -80,21 +97,160 @@ public class PlainTextExtractor implements FileExtractor { @Override public String extractText(DocumentSource source) throws IOException { - try (InputStream is = source.openStream()) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, "utf-8"))) { - StringBuilder text = new StringBuilder(); - char[] buffer = new char[8192]; - int read; - while ((read = reader.read(buffer)) != -1) { - text.append(buffer, 0, read); - } - return text.toString().trim(); + return read(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 按文本行区间读取 TXT 或 Markdown。 + * + * @param request 读取请求 + * @return 行区间结构化结果 + * @throws IOException 文本 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); + CharsetDetection detection = detectCharset(source); + try { + return readWithCharset(request, detection.charset(), detection.bomLength()); + } catch (CharacterCodingException error) { + if (!StandardCharsets.UTF_8.equals(detection.charset()) || detection.bomLength() > 0) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Text encoding is invalid", error); } - } catch (Exception e) { - throw new RuntimeException(e); + // 严格 UTF-8 解码失败时仅回退到受控 GB18030。 + return readWithCharset(request, GB18030, 0); + } catch (DocumentReadException error) { + throw error; + } catch (Exception error) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_READ_FAILED, + "Failed to read text document", error); } } + /** + * 使用指定字符集流式读取文本。 + * + * @param request 读取请求 + * @param charset 字符集 + * @param bomLength BOM 长度 + * @return 结构化结果 + * @throws Exception 打开或读取失败 + */ + private LightweightDocumentReadResult readWithCharset(LightweightDocumentReadRequest request, + Charset charset, + int bomLength) throws IOException { + DocumentSource source = request.getSource(); + List segments = new ArrayList<>(); + List headingPath = new ArrayList<>(); + boolean markdown = isMarkdown(source.getFileName(), source.getMimeType()); + try (InputStream input = openStream(source)) { + input.skipNBytes(bomLength); + InputStreamReader streamReader = new InputStreamReader(input, + charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT)); + try (BufferedReader reader = new BufferedReader(streamReader, 8192)) { + StringBuilder block = new StringBuilder(); + int line = 0; + int blockStart = 1; + int offset = 0; + String value; + while ((value = reader.readLine()) != null) { + request.checkCancelled(); + line++; + if (markdown) { + updateHeadingPath(headingPath, value); + } + if (!block.isEmpty()) { + block.append('\n'); + } + block.append(value); + if (line - blockStart + 1 >= LINES_PER_SEGMENT) { + DocumentTextSegment segment = addLineSegment( + segments, block, blockStart, line, offset, headingPath); + offset = segment == null ? offset : segment.getEndIndex() + 1; + block.setLength(0); + blockStart = line + 1; + } + if ((long) offset + block.length() > request.getMaxExpandedChars()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + "Expanded text exceeds the configured limit"); + } + } + if (!block.isEmpty()) { + addLineSegment(segments, block, blockStart, line, offset, headingPath); + } + } + } + return DocumentReadSupport.result(source, segments, request); + } + + private InputStream openStream(DocumentSource source) throws IOException { + try { + return source.openStream(); + } catch (IOException error) { + throw error; + } catch (Exception error) { + throw new IOException("Failed to open text document", error); + } + } + + private DocumentTextSegment addLineSegment(List segments, + StringBuilder block, + int startLine, + int endLine, + int offset, + List headingPath) { + if (block.toString().isBlank()) { + return null; + } + DocumentTextSegment segment = DocumentReadSupport.segment( + "lines-" + startLine + "-" + endLine, block.toString(), "LINE_RANGE", + "第 " + startLine + "-" + endLine + " 行", offset, headingPath); + segments.add(segment); + return segment; + } + + private void updateHeadingPath(List headingPath, String line) { + Matcher matcher = MARKDOWN_HEADING.matcher(line); + if (!matcher.matches()) { + return; + } + int level = matcher.group(1).length(); + while (headingPath.size() >= level) { + headingPath.remove(headingPath.size() - 1); + } + headingPath.add(matcher.group(2).trim()); + } + + private CharsetDetection detectCharset(DocumentSource source) throws IOException { + try (InputStream input = source.openStream()) { + byte[] prefix = input.readNBytes(3); + if (prefix.length >= 3 && (prefix[0] & 0xff) == 0xef + && (prefix[1] & 0xff) == 0xbb && (prefix[2] & 0xff) == 0xbf) { + return new CharsetDetection(StandardCharsets.UTF_8, 3); + } + if (prefix.length >= 2 && (prefix[0] & 0xff) == 0xff && (prefix[1] & 0xff) == 0xfe) { + return new CharsetDetection(StandardCharsets.UTF_16LE, 2); + } + if (prefix.length >= 2 && (prefix[0] & 0xff) == 0xfe && (prefix[1] & 0xff) == 0xff) { + return new CharsetDetection(StandardCharsets.UTF_16BE, 2); + } + return new CharsetDetection(StandardCharsets.UTF_8, 0); + } catch (IOException error) { + throw error; + } catch (Exception error) { + throw new IOException("Failed to inspect text encoding", error); + } + } + + private boolean isMarkdown(String fileName, String mimeType) { + return "text/markdown".equalsIgnoreCase(mimeType) + || (fileName != null && (fileName.toLowerCase().endsWith(".md") + || fileName.toLowerCase().endsWith(".markdown"))); + } + @Override public int getOrder() { @@ -106,4 +262,7 @@ public class PlainTextExtractor implements FileExtractor { int lastDot = fileName.lastIndexOf('.'); return fileName.substring(lastDot + 1).toLowerCase(); } + + private record CharsetDetection(Charset charset, int bomLength) { + } } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptExtractor.java new file mode 100644 index 0000000..5047489 --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptExtractor.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text.extractor.impl; + +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; +import com.easyagents.core.file2text.extractor.FileExtractor; +import com.easyagents.core.file2text.source.DocumentSource; +import org.apache.poi.hslf.usermodel.HSLFShape; +import org.apache.poi.hslf.usermodel.HSLFSlide; +import org.apache.poi.hslf.usermodel.HSLFSlideShow; +import org.apache.poi.hslf.usermodel.HSLFTextShape; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * PowerPoint 97-2003 文档提取器。 + */ +public class PptExtractor implements FileExtractor { + + private static final Set MIME_TYPES = Set.of( + "application/vnd.ms-powerpoint", + "application/mspowerpoint", + "application/powerpoint"); + + /** + * 判断是否支持 PPT。 + * + * @param source 文档来源 + * @return 是否支持 + */ + @Override + public boolean supports(DocumentSource source) { + if (source.getMimeType() != null && MIME_TYPES.contains(source.getMimeType().toLowerCase(Locale.ROOT))) { + return true; + } + String fileName = source.getFileName(); + return fileName != null && (fileName.toLowerCase(Locale.ROOT).endsWith(".ppt") + || fileName.toLowerCase(Locale.ROOT).endsWith(".pps")); + } + + /** + * 提取兼容纯文本。 + * + * @param source 文档来源 + * @return 文本 + * @throws IOException 文档 I/O 失败 + */ + @Override + public String extractText(DocumentSource source) throws IOException { + return read(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 按幻灯片读取 PPT 文本。 + * + * @param request 读取请求 + * @return 幻灯片结构化结果 + * @throws IOException 文档 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); + try (InputStream input = source.openStream(); + HSLFSlideShow slideShow = new HSLFSlideShow(input)) { + List slides = slideShow.getSlides(); + if (slides.size() > request.getMaxSlides()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + "Slide count exceeds " + request.getMaxSlides()); + } + List segments = new ArrayList<>(); + int offset = 0; + for (int index = 0; index < slides.size(); index++) { + request.checkCancelled(); + StringBuilder text = new StringBuilder(); + for (HSLFShape shape : slides.get(index).getShapes()) { + if (shape instanceof HSLFTextShape textShape) { + String value = textShape.getText(); + if (value != null && !value.isBlank()) { + text.append(value.trim()).append('\n'); + } + } + } + if (!text.toString().isBlank()) { + DocumentTextSegment segment = DocumentReadSupport.segment( + "slide-" + (index + 1), text.toString(), "SLIDE", + "第 " + (index + 1) + " 张幻灯片", offset, List.of()); + segments.add(segment); + offset = segment.getEndIndex() + 1; + } + } + return DocumentReadSupport.result(source, segments, request); + } catch (DocumentReadException error) { + throw error; + } catch (Exception error) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract PPT", error); + } + } + + /** + * 获取读取器优先级。 + * + * @return 优先级 + */ + @Override + public int getOrder() { + return 12; + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptxExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptxExtractor.java index bc2f085..8b73cfd 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptxExtractor.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/PptxExtractor.java @@ -15,6 +15,12 @@ */ package com.easyagents.core.file2text.extractor.impl; +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; import com.easyagents.core.file2text.extractor.FileExtractor; import com.easyagents.core.file2text.source.DocumentSource; import org.apache.poi.xslf.usermodel.*; @@ -77,21 +83,37 @@ public class PptxExtractor implements FileExtractor { @Override public String extractText(DocumentSource source) throws IOException { - StringBuilder text = new StringBuilder(); + return read(new LightweightDocumentReadRequest(source)).getText(); + } + /** + * 按幻灯片读取 PPTX 文本。 + * + * @param request 读取请求 + * @return 幻灯片结构化结果 + * @throws IOException 文档 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); try (InputStream is = source.openStream(); XMLSlideShow slideShow = new XMLSlideShow(is)) { List slides = slideShow.getSlides(); - + if (slides.size() > request.getMaxSlides()) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + "Slide count exceeds " + request.getMaxSlides()); + } + List segments = new ArrayList<>(); + int offset = 0; for (int i = 0; i < slides.size(); i++) { + request.checkCancelled(); XSLFSlide slide = slides.get(i); - text.append("\n--- Slide ").append(i + 1).append(" ---\n"); + StringBuilder text = new StringBuilder(); // 提取所有形状中的文本 for (XSLFShape shape : slide.getShapes()) { - if (shape instanceof XSLFTextShape) { - XSLFTextShape textShape = (XSLFTextShape) shape; + if (shape instanceof XSLFTextShape textShape && !(shape instanceof XSLFTable)) { String shapeText = textShape.getText(); if (shapeText != null && !shapeText.trim().isEmpty()) { text.append(shapeText).append("\n"); @@ -101,15 +123,24 @@ public class PptxExtractor implements FileExtractor { // 可选:提取表格 extractTablesFromSlide(slide, text); + if (!text.toString().isBlank()) { + DocumentTextSegment segment = DocumentReadSupport.segment( + "slide-" + (i + 1), text.toString(), "SLIDE", + "第 " + (i + 1) + " 张幻灯片", offset, List.of()); + segments.add(segment); + offset = segment.getEndIndex() + 1; + } } - + return DocumentReadSupport.result(source, segments, request); + } catch (DocumentReadException e) { + throw e; } catch (XmlException e) { - throw new IOException("Invalid PPTX structure: " + e.getMessage(), e); + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Invalid PPTX structure", e); } catch (Exception e) { - throw new IOException("Failed to extract PPTX: " + e.getMessage(), e); + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract PPTX", e); } - - return text.toString().trim(); } /** diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/SpreadsheetReadSupport.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/SpreadsheetReadSupport.java new file mode 100644 index 0000000..5ce3d8b --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/SpreadsheetReadSupport.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text.extractor.impl; + +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.DocumentTextSegment; + +import java.util.ArrayList; +import java.util.List; + +/** + * 表格行到稳定片段的内部转换工具。 + */ +final class SpreadsheetReadSupport { + + private static final int ROWS_PER_SEGMENT = 25; + + private SpreadsheetReadSupport() { + } + + /** + * 将工作表行按固定区间组成片段。 + * + * @param sheets 工作表数据 + * @return 文本片段 + */ + static List toSegments(List sheets) { + List segments = new ArrayList<>(); + int offset = 0; + for (SheetRows sheet : sheets) { + for (int start = 0; start < sheet.rows().size(); start += ROWS_PER_SEGMENT) { + int end = Math.min(sheet.rows().size(), start + ROWS_PER_SEGMENT); + List rows = sheet.rows().subList(start, end); + StringBuilder text = new StringBuilder(); + for (RowText row : rows) { + if (!text.isEmpty()) { + text.append('\n'); + } + text.append("第 ").append(row.rowNumber()).append(" 行: ").append(row.text()); + } + int startRow = rows.get(0).rowNumber(); + int endRow = rows.get(rows.size() - 1).rowNumber(); + DocumentTextSegment segment = DocumentReadSupport.segment( + "sheet-" + sheet.sheetIndex() + "-rows-" + startRow + "-" + endRow, + text.toString(), "SHEET_ROW_RANGE", + sheet.sheetName() + " 第 " + startRow + "-" + endRow + " 行", + offset, List.of(sheet.sheetName())); + segments.add(segment); + offset = segment.getEndIndex() + 1; + } + } + return segments; + } + + /** + * 单个工作表的有效行。 + * + * @param sheetIndex 工作表序号 + * @param sheetName 工作表名 + * @param rows 有效行 + */ + record SheetRows(int sheetIndex, String sheetName, List rows) { + } + + /** + * 单行格式化文本。 + * + * @param rowNumber 行号 + * @param text 文本 + */ + record RowText(int rowNumber, String text) { + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsExtractor.java new file mode 100644 index 0000000..3b00eb9 --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsExtractor.java @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text.extractor.impl; + +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; +import com.easyagents.core.file2text.extractor.FileExtractor; +import com.easyagents.core.file2text.source.DocumentSource; +import org.apache.poi.hssf.eventusermodel.FormatTrackingHSSFListener; +import org.apache.poi.hssf.eventusermodel.HSSFEventFactory; +import org.apache.poi.hssf.eventusermodel.HSSFListener; +import org.apache.poi.hssf.eventusermodel.HSSFRequest; +import org.apache.poi.hssf.record.BOFRecord; +import org.apache.poi.hssf.record.BoolErrRecord; +import org.apache.poi.hssf.record.BoundSheetRecord; +import org.apache.poi.hssf.record.FormulaRecord; +import org.apache.poi.hssf.record.LabelRecord; +import org.apache.poi.hssf.record.LabelSSTRecord; +import org.apache.poi.hssf.record.NumberRecord; +import org.apache.poi.hssf.record.Record; +import org.apache.poi.hssf.record.SSTRecord; +import org.apache.poi.hssf.record.StringRecord; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.ss.util.CellReference; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * 基于 HSSF Event API 的 XLS 文档提取器。 + */ +public class XlsExtractor implements FileExtractor { + + private static final Set MIME_TYPES = Set.of( + "application/vnd.ms-excel", + "application/msexcel", + "application/x-msexcel"); + + /** + * 判断是否支持 XLS。 + * + * @param source 文档来源 + * @return 是否支持 + */ + @Override + public boolean supports(DocumentSource source) { + if (source.getMimeType() != null && MIME_TYPES.contains(source.getMimeType().toLowerCase(Locale.ROOT))) { + return true; + } + String name = source.getFileName(); + return name != null && (name.toLowerCase(Locale.ROOT).endsWith(".xls") + || name.toLowerCase(Locale.ROOT).endsWith(".xlt")); + } + + /** + * 提取兼容纯文本。 + * + * @param source 文档来源 + * @return 文本 + * @throws IOException 文档 I/O 失败 + */ + @Override + public String extractText(DocumentSource source) throws IOException { + return read(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 使用 HSSF 事件模型按工作表和行读取 XLS。 + * + * @param request 读取请求 + * @return 表格结构化结果 + * @throws IOException 文档 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); + try (InputStream input = source.openStream(); + POIFSFileSystem fileSystem = new POIFSFileSystem(input)) { + XlsListener listener = new XlsListener(request); + FormatTrackingHSSFListener formatter = new FormatTrackingHSSFListener(listener); + listener.setFormatter(formatter); + HSSFRequest hssfRequest = new HSSFRequest(); + hssfRequest.addListenerForAllRecords(formatter); + new HSSFEventFactory().processWorkbookEvents(hssfRequest, fileSystem); + listener.finish(); + return DocumentReadSupport.result(source, + SpreadsheetReadSupport.toSegments(listener.sheets()), request); + } catch (StructureLimitRuntimeException error) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, + error.getMessage(), error); + } catch (DocumentReadException error) { + throw error; + } catch (Exception error) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract XLS", error); + } + } + + /** + * 获取读取器优先级。 + * + * @return 优先级 + */ + @Override + public int getOrder() { + return 10; + } + + /** + * HSSF 二进制记录监听器。 + */ + private static final class XlsListener implements HSSFListener { + + private final LightweightDocumentReadRequest request; + private final List boundSheets = new ArrayList<>(); + private final List sheets = new ArrayList<>(); + private final List currentRows = new ArrayList<>(); + private final Map currentCells = new LinkedHashMap<>(); + private FormatTrackingHSSFListener formatter; + private SSTRecord sharedStrings; + private int sheetIndex; + private int currentRow = -1; + private int nonEmptyCells; + private int pendingFormulaRow = -1; + private int pendingFormulaColumn = -1; + + private XlsListener(LightweightDocumentReadRequest request) { + this.request = request; + } + + private void setFormatter(FormatTrackingHSSFListener formatter) { + this.formatter = formatter; + } + + /** + * 处理一个 HSSF 记录。 + * + * @param record 工作簿记录 + */ + @Override + public void processRecord(Record record) { + checkCancelled(); + if (record instanceof BoundSheetRecord boundSheet) { + boundSheets.add(boundSheet); + } else if (record instanceof SSTRecord sstRecord) { + sharedStrings = sstRecord; + } else if (record instanceof BOFRecord bofRecord + && bofRecord.getType() == BOFRecord.TYPE_WORKSHEET) { + startSheet(); + } else if (record instanceof LabelSSTRecord label) { + String value = sharedStrings == null ? "" : sharedStrings.getString(label.getSSTIndex()).toString(); + putCell(label.getRow(), label.getColumn(), value); + } else if (record instanceof LabelRecord label) { + putCell(label.getRow(), label.getColumn(), label.getValue()); + } else if (record instanceof NumberRecord number) { + putCell(number.getRow(), number.getColumn(), formatter.formatNumberDateCell(number)); + } else if (record instanceof FormulaRecord formula) { + if (formula.hasCachedResultString()) { + pendingFormulaRow = formula.getRow(); + pendingFormulaColumn = formula.getColumn(); + } else { + putCell(formula.getRow(), formula.getColumn(), formatter.formatNumberDateCell(formula)); + } + } else if (record instanceof StringRecord string && pendingFormulaRow >= 0) { + putCell(pendingFormulaRow, pendingFormulaColumn, string.getString()); + pendingFormulaRow = -1; + pendingFormulaColumn = -1; + } else if (record instanceof BoolErrRecord boolError && boolError.isBoolean()) { + putCell(boolError.getRow(), boolError.getColumn(), + Boolean.toString(boolError.getBooleanValue())); + } + } + + private void startSheet() { + flushSheet(); + sheetIndex++; + if (sheetIndex > request.getMaxSheets()) { + throw new StructureLimitRuntimeException( + "Sheet count exceeds " + request.getMaxSheets()); + } + } + + private void putCell(int row, int column, String value) { + if (value == null || value.isBlank()) { + return; + } + if (currentRow >= 0 && row != currentRow) { + flushRow(); + } + currentRow = row; + nonEmptyCells++; + if (nonEmptyCells > request.getMaxNonEmptyCells()) { + throw new StructureLimitRuntimeException( + "Non-empty cell count exceeds " + request.getMaxNonEmptyCells()); + } + currentCells.put(column, + CellReference.convertNumToColString(column) + (row + 1) + "=" + value.trim()); + } + + private void flushRow() { + if (!currentCells.isEmpty() && currentRow >= 0) { + String text = String.join(" | ", currentCells.values()); + currentRows.add(new SpreadsheetReadSupport.RowText(currentRow + 1, text)); + } + currentCells.clear(); + currentRow = -1; + } + + private void flushSheet() { + flushRow(); + if (sheetIndex <= 0) { + return; + } + String name = sheetIndex <= boundSheets.size() + ? boundSheets.get(sheetIndex - 1).getSheetname() + : "Sheet " + sheetIndex; + sheets.add(new SpreadsheetReadSupport.SheetRows( + sheetIndex, name, new ArrayList<>(currentRows))); + currentRows.clear(); + } + + private void finish() { + flushSheet(); + } + + private void checkCancelled() { + request.checkCancelled(); + } + + private List sheets() { + return sheets; + } + } + + /** + * HSSF 回调中传递读取中止或结构上限异常。 + */ + private static final class StructureLimitRuntimeException extends RuntimeException { + + private StructureLimitRuntimeException(String message) { + super(message); + } + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsxExtractor.java b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsxExtractor.java new file mode 100644 index 0000000..65e727d --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/file2text/extractor/impl/XlsxExtractor.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0. + */ +package com.easyagents.core.file2text.extractor.impl; + +import com.easyagents.core.file2text.DocumentReadErrorCode; +import com.easyagents.core.file2text.DocumentReadException; +import com.easyagents.core.file2text.DocumentReadSupport; +import com.easyagents.core.file2text.LightweightDocumentReadRequest; +import com.easyagents.core.file2text.LightweightDocumentReadResult; +import com.easyagents.core.file2text.extractor.FileExtractor; +import com.easyagents.core.file2text.source.DocumentSource; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.util.CellReference; +import org.apache.poi.util.XMLHelper; +import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler; +import org.apache.poi.xssf.model.SharedStrings; +import org.apache.poi.xssf.model.Styles; +import org.apache.poi.xssf.usermodel.XSSFComment; +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * 基于 XSSF SAX 的 XLSX 文档提取器。 + */ +public class XlsxExtractor implements FileExtractor { + + private static final Set MIME_TYPES = Set.of( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template"); + + /** + * 判断是否支持 XLSX。 + * + * @param source 文档来源 + * @return 是否支持 + */ + @Override + public boolean supports(DocumentSource source) { + if (source.getMimeType() != null && MIME_TYPES.contains(source.getMimeType().toLowerCase(Locale.ROOT))) { + return true; + } + String name = source.getFileName(); + return name != null && (name.toLowerCase(Locale.ROOT).endsWith(".xlsx") + || name.toLowerCase(Locale.ROOT).endsWith(".xltx")); + } + + /** + * 提取兼容纯文本。 + * + * @param source 文档来源 + * @return 文本 + * @throws IOException 文档 I/O 失败 + */ + @Override + public String extractText(DocumentSource source) throws IOException { + return read(new LightweightDocumentReadRequest(source)).getText(); + } + + /** + * 使用 SAX 按工作表和行读取 XLSX。 + * + * @param request 读取请求 + * @return 表格结构化结果 + * @throws IOException 文档 I/O 失败 + */ + @Override + public LightweightDocumentReadResult read(LightweightDocumentReadRequest request) throws IOException { + DocumentSource source = request.getSource(); + try (InputStream input = source.openStream(); + OPCPackage opcPackage = OPCPackage.open(input)) { + XSSFReader reader = new XSSFReader(opcPackage); + Styles styles = reader.getStylesTable(); + SharedStrings sharedStrings = new ReadOnlySharedStringsTable(opcPackage); + XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) reader.getSheetsData(); + List resultSheets = new ArrayList<>(); + int[] nonEmptyCells = {0}; + int sheetIndex = 0; + while (sheets.hasNext()) { + request.checkCancelled(); + sheetIndex++; + if (sheetIndex > request.getMaxSheets()) { + throw limit("Sheet count exceeds " + request.getMaxSheets()); + } + try (InputStream sheetInput = sheets.next()) { + String sheetName = sheets.getSheetName(); + SheetHandler handler = new SheetHandler(request, nonEmptyCells); + XMLReader parser = XMLHelper.newXMLReader(); + parser.setContentHandler(new XSSFSheetXMLHandler( + styles, null, sharedStrings, handler, new DataFormatter(), false)); + parser.parse(new InputSource(sheetInput)); + resultSheets.add(new SpreadsheetReadSupport.SheetRows( + sheetIndex, sheetName, handler.rows())); + } + } + return DocumentReadSupport.result(source, + SpreadsheetReadSupport.toSegments(resultSheets), request); + } catch (StructureLimitRuntimeException error) { + throw limit(error.getMessage()); + } catch (DocumentReadException error) { + throw error; + } catch (Exception error) { + throw new DocumentReadException(DocumentReadErrorCode.DOCUMENT_CORRUPTED, + "Failed to extract XLSX", error); + } + } + + /** + * 获取读取器优先级。 + * + * @return 优先级 + */ + @Override + public int getOrder() { + return 10; + } + + private DocumentReadException limit(String message) { + return new DocumentReadException(DocumentReadErrorCode.DOCUMENT_STRUCTURE_LIMIT_EXCEEDED, message); + } + + /** + * SAX 工作表内容处理器。 + */ + private static final class SheetHandler implements XSSFSheetXMLHandler.SheetContentsHandler { + + private final LightweightDocumentReadRequest request; + private final int[] nonEmptyCells; + private final List rows = new ArrayList<>(); + private final Map currentCells = new LinkedHashMap<>(); + private int currentRow; + + private SheetHandler(LightweightDocumentReadRequest request, int[] nonEmptyCells) { + this.request = request; + this.nonEmptyCells = nonEmptyCells; + } + + /** + * 开始读取一行。 + * + * @param rowNum 零基行号 + */ + @Override + public void startRow(int rowNum) { + currentRow = rowNum + 1; + currentCells.clear(); + } + + /** + * 完成一行并保存有效单元格。 + * + * @param rowNum 零基行号 + */ + @Override + public void endRow(int rowNum) { + if (currentCells.isEmpty()) { + return; + } + String text = currentCells.entrySet().stream() + .map(item -> item.getKey() + "=" + item.getValue()) + .reduce((left, right) -> left + " | " + right) + .orElse(""); + rows.add(new SpreadsheetReadSupport.RowText(currentRow, text)); + } + + /** + * 接收一个格式化单元格值。 + * + * @param cellReference 单元格引用 + * @param formattedValue 格式化显示值 + * @param comment 批注 + */ + @Override + public void cell(String cellReference, String formattedValue, XSSFComment comment) { + if (formattedValue == null || formattedValue.isBlank()) { + return; + } + request.checkCancelled(); + nonEmptyCells[0]++; + if (nonEmptyCells[0] > request.getMaxNonEmptyCells()) { + throw new StructureLimitRuntimeException( + "Non-empty cell count exceeds " + request.getMaxNonEmptyCells()); + } + String column = cellReference == null ? "?" : CellReference.convertNumToColString( + new CellReference(cellReference).getCol()); + currentCells.put(column + currentRow, formattedValue.trim()); + } + + /** + * 接收页眉页脚;轻量读取不纳入正文。 + * + * @param text 文本 + * @param isHeader 是否页眉 + * @param tagName 标签名 + */ + @Override + public void headerFooter(String text, boolean isHeader, String tagName) { + } + + private List rows() { + return rows; + } + } + + /** + * SAX 回调跨层传递结构上限异常。 + */ + private static final class StructureLimitRuntimeException extends RuntimeException { + + private StructureLimitRuntimeException(String message) { + super(message); + } + } +} diff --git a/easy-agents-core/src/test/java/com/easyagents/core/file2text/File2TextServiceLightweightReadTest.java b/easy-agents-core/src/test/java/com/easyagents/core/file2text/File2TextServiceLightweightReadTest.java new file mode 100644 index 0000000..2f7dee5 --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/file2text/File2TextServiceLightweightReadTest.java @@ -0,0 +1,162 @@ +package com.easyagents.core.file2text; + +import com.easyagents.core.file2text.extractor.impl.DocExtractor; +import com.easyagents.core.file2text.extractor.impl.PdfTextExtractor; +import com.easyagents.core.file2text.extractor.impl.XlsExtractor; +import com.easyagents.core.file2text.extractor.impl.XlsxExtractor; +import com.easyagents.core.file2text.source.ByteArrayDocumentSource; +import org.apache.poi.hslf.usermodel.HSLFSlide; +import org.apache.poi.hslf.usermodel.HSLFSlideShow; +import org.apache.poi.hslf.usermodel.HSLFTextBox; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.xslf.usermodel.XMLSlideShow; +import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +/** + * 常规文档格式轻量读取回归测试。 + */ +public class File2TextServiceLightweightReadTest { + + private final File2TextService service = new File2TextService(); + + /** + * 验证 TXT 与 Markdown 会保留行区间和标题结构。 + */ + @Test + public void shouldReadTextAndMarkdownWithStableSegments() { + LightweightDocumentReadResult text = read( + "sample.txt", "text/plain", "first line\nsecond line".getBytes(StandardCharsets.UTF_8)); + LightweightDocumentReadResult markdown = read( + "sample.md", "text/markdown", "# Chapter\nbody".getBytes(StandardCharsets.UTF_8)); + + Assert.assertTrue(text.getText().contains("second line")); + Assert.assertEquals("LINE_RANGE", text.getSegments().get(0).getLocatorType()); + Assert.assertTrue(markdown.getText().contains("Chapter")); + Assert.assertEquals("Chapter", markdown.getSegments().get(0).getHeadingPath().get(0)); + } + + /** + * 验证 DOCX、PPTX、PPT、XLSX 与 XLS 均可直接读取正文。 + * + * @throws Exception 测试文档生成失败时抛出 + */ + @Test + public void shouldReadGeneratedOfficeAndPdfDocuments() throws Exception { + assertReadable("sample.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + docxBytes(), "DOCX sample"); + assertReadable("sample.pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + pptxBytes(), "PPTX sample"); + assertReadable("sample.ppt", "application/vnd.ms-powerpoint", pptBytes(), "PPT sample"); + assertReadable("sample.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + xlsxBytes(), "XLSX sample"); + assertReadable("sample.xls", "application/vnd.ms-excel", xlsBytes(), "XLS sample"); + } + + /** + * 验证 PDF 与旧版 DOC 扩展名仍路由到专用读取器。 + */ + @Test + public void shouldRegisterPdfAndLegacyDocReaders() { + DocExtractor docExtractor = new DocExtractor(); + PdfTextExtractor pdfExtractor = new PdfTextExtractor(); + + Assert.assertTrue(docExtractor.supports(new ByteArrayDocumentSource( + new byte[0], "legacy.doc", "application/msword"))); + Assert.assertTrue(pdfExtractor.supports(new ByteArrayDocumentSource( + new byte[0], "sample.pdf", "application/pdf"))); + } + + /** + * 验证表格读取取消会保留明确错误码。 + * + * @throws Exception 测试文档生成失败时抛出 + */ + @Test + public void shouldPreserveCancellationErrorForSpreadsheets() throws Exception { + assertCancelled(new XlsxExtractor(), "sample.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", xlsxBytes()); + assertCancelled(new XlsExtractor(), "sample.xls", + "application/vnd.ms-excel", xlsBytes()); + } + + private void assertCancelled(com.easyagents.core.file2text.extractor.FileExtractor extractor, + String fileName, + String mimeType, + byte[] bytes) throws Exception { + LightweightDocumentReadRequest request = new LightweightDocumentReadRequest( + new ByteArrayDocumentSource(bytes, fileName, mimeType)); + request.setCancelled(() -> true); + try { + extractor.read(request); + Assert.fail("Expected document read cancellation"); + } catch (DocumentReadException error) { + Assert.assertEquals(DocumentReadErrorCode.DOCUMENT_READ_CANCELLED, error.getErrorCode()); + } + } + + private void assertReadable(String fileName, String mimeType, byte[] bytes, String expected) { + Assert.assertTrue(read(fileName, mimeType, bytes).getText().contains(expected)); + } + + private LightweightDocumentReadResult read(String fileName, String mimeType, byte[] bytes) { + return service.readFromStream(new ByteArrayInputStream(bytes), fileName, mimeType); + } + + private byte[] docxBytes() throws Exception { + try (XWPFDocument document = new XWPFDocument(); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + document.createParagraph().createRun().setText("DOCX sample"); + document.write(output); + return output.toByteArray(); + } + } + + private byte[] pptxBytes() throws Exception { + try (XMLSlideShow presentation = new XMLSlideShow(); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + presentation.createSlide().createTextBox().setText("PPTX sample"); + presentation.write(output); + return output.toByteArray(); + } + } + + private byte[] pptBytes() throws Exception { + try (HSLFSlideShow presentation = new HSLFSlideShow(); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + HSLFSlide slide = presentation.createSlide(); + HSLFTextBox textBox = new HSLFTextBox(); + textBox.setText("PPT sample"); + slide.addShape(textBox); + presentation.write(output); + return output.toByteArray(); + } + } + + private byte[] xlsxBytes() throws Exception { + try (XSSFWorkbook workbook = new XSSFWorkbook(); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + workbook.createSheet("Sheet1").createRow(0).createCell(0).setCellValue("XLSX sample"); + workbook.write(output); + return output.toByteArray(); + } + } + + private byte[] xlsBytes() throws Exception { + try (HSSFWorkbook workbook = new HSSFWorkbook(); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + workbook.createSheet("Sheet1").createRow(0).createCell(0).setCellValue("XLS sample"); + workbook.write(output); + return output.toByteArray(); + } + } +} From 851dd1be01ba1974e7dae4eb46034b46b798b099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 29 Jul 2026 18:05:23 +0800 Subject: [PATCH 14/33] =?UTF-8?q?feat:=20=E5=A2=9E=E5=BC=BA=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E6=B5=81=E5=BE=AA=E7=8E=AF=E5=8F=8C=E8=BE=93=E5=85=A5?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E8=AF=AD=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持次数、数组及二者组合的循环执行计划 - 保持旧循环定义和上游数组读取兼容 - 补充解析、边界与前缀遍历测试 --- .../easyagents/flow/core/node/LoopNode.java | 404 +++++++++++++++--- .../flow/core/parser/impl/LoopNodeParser.java | 40 +- .../flow/core/test/LoopNodeParserTest.java | 95 ++++ .../test/LoopNodeProgressContextTest.java | 214 ++++++++++ 4 files changed, 695 insertions(+), 58 deletions(-) create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeParserTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java index ff7f2bf..7d5895f 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java @@ -35,6 +35,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +/** + * 支持纯次数、数组遍历和两者组合限制的显式循环节点。 + */ public class LoopNode extends BaseNode { private static final long serialVersionUID = 1L; @@ -43,16 +46,70 @@ public class LoopNode extends BaseNode { Integer.getInteger( "tinyflow.loop.direct-index.max-items", 64)); + /** 旧版单循环参数。 */ private Parameter loopVar; + /** 可选的循环次数参数。 */ + private Parameter loopCount; + /** 可选的数组输入参数。 */ + private Parameter loopItems; + /** + * 获取旧版循环参数。 + * + * @return 旧版循环参数 + */ public Parameter getLoopVar() { return loopVar; } + /** + * 设置旧版循环参数。 + * + * @param loopVar 旧版循环参数 + */ public void setLoopVar(Parameter loopVar) { this.loopVar = loopVar; } + /** + * 获取显式循环次数参数。 + * + * @return 循环次数参数 + */ + public Parameter getLoopCount() { + return loopCount; + } + + /** + * 设置显式循环次数参数。 + * + * @param loopCount 循环次数参数 + */ + public void setLoopCount(Parameter loopCount) { + this.loopCount = loopCount; + } + + /** + * 获取待遍历数组参数。 + * + * @return 数组参数 + */ + public Parameter getLoopItems() { + return loopItems; + } + + /** + * 设置待遍历数组参数。 + * + * @param loopItems 数组参数 + */ + public void setLoopItems(Parameter loopItems) { + this.loopItems = loopItems; + } + + /** + * {@inheritDoc} + */ @Override public Map execute(Chain chain) { MaterializationPlan[] planHolder = new MaterializationPlan[1]; @@ -164,9 +221,17 @@ public class LoopNode extends BaseNode { boolean directlyIndexed = false; boolean numericLoop = false; Object loopValue = null; + boolean usesExplicitInputs = + loopCount != null || loopItems != null; + Integer requestedCount = usesExplicitInputs + ? resolveRequestedLoopCount(chain, loopContext) + : null; if (loopContext.iterableInputStored) { // 已物化输入不再解析原始参数,避免大集合在每轮反序列化和路径解析。 - shouldLoopCount = loopContext.iterableSize; + shouldLoopCount = planIterableIterations( + loopContext, + loopContext.iterableSize, + requestedCount); storedIterable = true; } else { LoopInputReference storedInput = @@ -178,52 +243,117 @@ public class LoopNode extends BaseNode { storedInput.getItemCount(); loopContext.iterableInputStored = true; loopContext.inputExternalized = true; - shouldLoopCount = - storedInput.getItemCount(); + shouldLoopCount = planIterableIterations( + loopContext, + storedInput.getItemCount(), + requestedCount); storedIterable = true; persistLoopStack(chain, loopStack); - } else { - Map loopVars = - chain.getExecutionState().resolveParameters( - this, - Collections.singletonList( - loopVar)); - loopValue = loopVars.get(loopVar.getName()); + } else if (loopItems != null) { + loopValue = resolveParameterValue( + chain, loopItems, "loop items"); + Iterable iterableInput = + toIterableInput(loopValue); + if (iterableInput == null) { + throw invalidLoopItemsValue(loopValue); + } + int knownSize = knownInputSize(loopValue); + if (knownSize >= 0) { + shouldLoopCount = planIterableIterations( + loopContext, + knownSize, + requestedCount); + checkExplicitLoopIterations( + chain, shouldLoopCount, true); + } else { + shouldLoopCount = requestedCount == null + ? -1 + : requestedCount; + } + + if (knownSize >= 0 + && shouldLoopCount <= DIRECT_INDEX_MAX_ITEMS) { + directlyIndexed = true; + } else { + /* + * 有次数限制时只物化所需前缀。前缀快照仅属于当前循环, + * 不替换上游完整数组,避免影响其他下游节点。 + */ + Iterable materializedInput = + requestedCount == null + ? iterableInput + : limitIterable( + iterableInput, + requestedCount); + boolean externalizeInput = + requestedCount == null + || (knownSize >= 0 + && requestedCount >= knownSize); + loopContext.resultId = + chain.getStateInstanceId() + ":" + UUID.randomUUID(); + loopContext.materializingInput = true; + loopContext.materializationClaimId = + chain.currentFencingClaimId(); + loopContext.materializationClaimGeneration = + chain.currentClaimGeneration(); + persistLoopStack(chain, loopStack); + if (planHolder == null) { + throw new IllegalStateException( + "Loop materialization plan holder is unavailable"); + } + planHolder[0] = new MaterializationPlan( + loopContext.resultId, + materializedInput, + loopItems, + externalizeInput, + loopContext.materializationClaimId, + loopContext.materializationClaimGeneration); + return waitingResult(); + } + } else if (loopCount != null) { + shouldLoopCount = requestedCount; + numericLoop = true; + planNumericIterations(loopContext, shouldLoopCount); + } else if (loopVar != null) { + loopValue = resolveParameterValue( + chain, loopVar, "legacy loop value"); Iterable iterableInput = toIterableInput(loopValue); if (iterableInput != null) { - int knownSize = knownInputSize(loopValue); - if (knownSize >= 0) { - checkExplicitLoopIterations(chain, knownSize, true); - } - if (knownSize >= 0 - && knownSize <= DIRECT_INDEX_MAX_ITEMS) { - shouldLoopCount = knownSize; - directlyIndexed = true; - } else { - /* - * 每次未完成的物化尝试使用新结果 ID。失锁 owner 的旧分块仅会按 TTL - * 回收,接管者不会复用或删除其部分数据。 - */ - loopContext.resultId = - chain.getStateInstanceId() + ":" + UUID.randomUUID(); - loopContext.materializingInput = true; - loopContext.materializationClaimId = - chain.currentFencingClaimId(); - loopContext.materializationClaimGeneration = - chain.currentClaimGeneration(); - persistLoopStack(chain, loopStack); - if (planHolder == null) { - throw new IllegalStateException( - "Loop materialization plan holder is unavailable"); - } - planHolder[0] = new MaterializationPlan( - loopContext.resultId, - iterableInput, - loopContext.materializationClaimId, - loopContext.materializationClaimGeneration); - return waitingResult(); - } + int knownSize = knownInputSize(loopValue); + if (knownSize >= 0) { + checkExplicitLoopIterations(chain, knownSize, true); + } + if (knownSize >= 0 + && knownSize <= DIRECT_INDEX_MAX_ITEMS) { + shouldLoopCount = knownSize; + directlyIndexed = true; + } else { + /* + * 每次未完成的物化尝试使用新结果 ID。失锁 owner 的旧分块仅会按 TTL + * 回收,接管者不会复用或删除其部分数据。 + */ + loopContext.resultId = + chain.getStateInstanceId() + ":" + UUID.randomUUID(); + loopContext.materializingInput = true; + loopContext.materializationClaimId = + chain.currentFencingClaimId(); + loopContext.materializationClaimGeneration = + chain.currentClaimGeneration(); + persistLoopStack(chain, loopStack); + if (planHolder == null) { + throw new IllegalStateException( + "Loop materialization plan holder is unavailable"); + } + planHolder[0] = new MaterializationPlan( + loopContext.resultId, + iterableInput, + loopVar, + true, + loopContext.materializationClaimId, + loopContext.materializationClaimGeneration); + return waitingResult(); + } } else if (loopValue instanceof Number || loopValue instanceof String) { shouldLoopCount = parseNumericLoopCount(loopValue); @@ -231,6 +361,9 @@ public class LoopNode extends BaseNode { } else { throw invalidLoopValue(loopValue); } + } else { + throw new IllegalArgumentException( + "Loop node requires loop count or loop items"); } } checkExplicitLoopIterations(chain, shouldLoopCount, !numericLoop); @@ -329,10 +462,16 @@ public class LoopNode extends BaseNode { "loop-materialization:" + plan.resultId); } context.iterableSize = iterableSize; + context.plannedIterations = iterableSize; + context.plannedIterationsResolved = true; context.iterableInputStored = true; context.materializingInput = false; - context.inputExternalized = externalizeMaterializedInput( - chain, plan.resultId, iterableSize); + context.inputExternalized = plan.externalizeInput + && externalizeMaterializedInput( + chain, + plan.sourceParameter, + plan.resultId, + iterableSize); persistLoopStack(chain, loopStack); return executeLocked(chain, true, null); } @@ -349,9 +488,16 @@ public class LoopNode extends BaseNode { * @return 已替换热状态值时为 {@code true} */ private boolean externalizeMaterializedInput( - Chain chain, String resultId, int iterableSize) { - String ref = loopVar == null ? null : loopVar.getRef(); - String name = loopVar == null ? null : loopVar.getName(); + Chain chain, + Parameter sourceParameter, + String resultId, + int iterableSize) { + String ref = sourceParameter == null + ? null + : sourceParameter.getRef(); + String name = sourceParameter == null + ? null + : sourceParameter.getName(); AtomicBoolean replaced = new AtomicBoolean(); chain.updateStateSafely(state -> { ConcurrentHashMap memory = @@ -389,18 +535,20 @@ public class LoopNode extends BaseNode { */ private LoopInputReference resolveStoredInputReference( Chain chain) { - if (loopVar == null) { + Parameter sourceParameter = + loopItems != null ? loopItems : loopVar; + if (sourceParameter == null) { return null; } Map memory = chain.getExecutionState().getMemory(); - String ref = loopVar.getRef(); + String ref = sourceParameter.getRef(); if (StringUtil.hasText(ref) && memory.get(ref) instanceof LoopInputReference) { return (LoopInputReference) memory.get(ref); } - String name = loopVar.getName(); + String name = sourceParameter.getName(); return StringUtil.hasText(name) && memory.get(name) instanceof LoopInputReference @@ -418,6 +566,119 @@ public class LoopNode extends BaseNode { .set(ChainConsts.NODE_STATE_STATUS_KEY, NodeStatus.RUNNING); } + /** + * 解析新结构中的循环次数,并在当前循环上下文中固定结果。 + * + * @param chain 当前工作流 + * @param context 当前循环上下文 + * @return 已配置次数;未配置时返回 {@code null} + */ + private Integer resolveRequestedLoopCount( + Chain chain, LoopContext context) { + if (loopCount == null) { + return null; + } + if (context.requestedCountResolved) { + return context.requestedCount; + } + int count = parseNumericLoopCount( + resolveParameterValue( + chain, loopCount, "loop count")); + context.requestedCount = count; + context.requestedCountResolved = true; + return count; + } + + /** + * 解析单个循环参数。 + * + * @param chain 当前工作流 + * @param parameter 待解析参数 + * @param parameterLabel 错误提示中的参数名称 + * @return 参数运行值 + */ + private Object resolveParameterValue( + Chain chain, + Parameter parameter, + String parameterLabel) { + if (parameter == null + || !StringUtil.hasText(parameter.getName())) { + throw new IllegalArgumentException( + parameterLabel + " parameter is missing"); + } + Map values = + chain.getExecutionState().resolveParameters( + this, + Collections.singletonList(parameter)); + return values.get(parameter.getName()); + } + + /** + * 固定数组循环的实际迭代次数。 + * + * @param context 当前循环上下文 + * @param iterableSize 数组实际元素数 + * @param requestedCount 可选的最大处理数量 + * @return 本轮实际迭代次数 + */ + private int planIterableIterations( + LoopContext context, + int iterableSize, + Integer requestedCount) { + if (context.plannedIterationsResolved) { + return context.plannedIterations; + } + int iterations = requestedCount == null + ? iterableSize + : Math.min(requestedCount, iterableSize); + context.plannedIterations = iterations; + context.plannedIterationsResolved = true; + return iterations; + } + + /** + * 固定纯次数循环的实际迭代次数。 + * + * @param context 当前循环上下文 + * @param iterations 迭代次数 + */ + private void planNumericIterations( + LoopContext context, int iterations) { + if (!context.plannedIterationsResolved) { + context.plannedIterations = iterations; + context.plannedIterationsResolved = true; + } + } + + /** + * 将可迭代输入限制为最多读取指定数量元素。 + * + * @param source 原始可迭代输入 + * @param limit 最大元素数 + * @return 单次消费的有界可迭代输入 + */ + private Iterable limitIterable( + Iterable source, int limit) { + return () -> new Iterator() { + private final Iterator delegate = source.iterator(); + private int remaining = limit; + + @Override + public boolean hasNext() { + return remaining > 0 && delegate.hasNext(); + } + + @Override + public Object next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + remaining--; + return delegate.next(); + } + }; + } + /** * 将集合或数组统一转换为单次消费的 Iterable。 * @@ -577,6 +838,22 @@ public class LoopNode extends BaseNode { + "\""); } + /** + * 创建输入数组类型错误。 + * + * @param loopValue 非法数组输入 + * @return 参数异常 + */ + private IllegalArgumentException invalidLoopItemsValue( + Object loopValue) { + String actualType = loopValue == null + ? "null" + : loopValue.getClass().getName(); + return new IllegalArgumentException( + "Loop items must resolve to Iterable or array, but actual type is " + + actualType); + } + /** * 获取或创建当前节点的 LoopContext 堆栈(每个 LoopNode 实例独立) @@ -856,6 +1133,14 @@ public class LoopNode extends BaseNode { long materializationClaimGeneration; int iterableSize; long accumulatedBytes; + /** 是否已固定显式次数输入。 */ + boolean requestedCountResolved; + /** 当前循环固定后的次数输入。 */ + int requestedCount; + /** 是否已固定实际迭代次数。 */ + boolean plannedIterationsResolved; + /** 当前循环固定后的实际迭代次数。 */ + int plannedIterations; int expectedReturnCount = 1; Set completedBranchIds = new LinkedHashSet<>(); @@ -940,6 +1225,17 @@ public class LoopNode extends BaseNode { this.iterableSize = iterableSize; } + /** + * 获取当前循环固定后的实际迭代次数。 + * + * @return 实际迭代次数;尚未规划时返回 {@code null} + */ + public Integer getPlannedIterations() { + return plannedIterationsResolved + ? plannedIterations + : null; + } + public int getExpectedReturnCount() { return expectedReturnCount; } @@ -968,6 +1264,8 @@ public class LoopNode extends BaseNode { private final String resultId; private final Iterable items; + private final Parameter sourceParameter; + private final boolean externalizeInput; private final String claimId; private final long claimGeneration; @@ -976,16 +1274,22 @@ public class LoopNode extends BaseNode { * * @param resultId 唯一结果 ID * @param items 输入元素 + * @param sourceParameter 输入来源参数 + * @param externalizeInput 是否允许替换上游完整输入 * @param claimId 触发器 ID * @param claimGeneration 触发器代际 */ private MaterializationPlan( String resultId, Iterable items, + Parameter sourceParameter, + boolean externalizeInput, String claimId, long claimGeneration) { this.resultId = resultId; this.items = items; + this.sourceParameter = sourceParameter; + this.externalizeInput = externalizeInput; this.claimId = claimId; this.claimGeneration = claimGeneration; } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/LoopNodeParser.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/LoopNodeParser.java index 3a42639..5cef0cb 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/LoopNodeParser.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/LoopNodeParser.java @@ -15,6 +15,7 @@ */ package com.easyagents.flow.core.parser.impl; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.node.LoopNode; @@ -22,26 +23,49 @@ import com.easyagents.flow.core.parser.BaseNodeParser; import java.util.List; +/** + * 解析显式循环节点的双输入与旧版单输入配置。 + */ public class LoopNodeParser extends BaseNodeParser { + /** + * {@inheritDoc} + */ @Override public LoopNode doParse(JSONObject root, JSONObject data, JSONObject chainJSONObject) { LoopNode loopNode = new LoopNode(); - // 这里需要设置 id,先设置 id 后, loopNode.setLoopChain(chain); 才能取获取当前节点的 id -// loopNode.setId(root.getString("id")); + JSONObject loopInputs = data.getJSONObject("loopInputs"); + if (loopInputs != null) { + loopNode.setLoopCount( + parseParameter(loopInputs.getJSONObject("count"))); + loopNode.setLoopItems( + parseParameter(loopInputs.getJSONObject("items"))); + return loopNode; + } + // 新结构不存在时继续兼容旧版 loopVars[0]。 List loopVars = getParameters(data, "loopVars"); if (!loopVars.isEmpty()) { loopNode.setLoopVar(loopVars.get(0)); } -// JSONArray nodes = chainJSONObject.getJSONArray("nodes"); -// JSONArray edges = chainJSONObject.getJSONArray("edges"); - -// ChainDefinition chain = getChainParser().parse(chainJSONObject, nodes, edges, root); -// loopNode.setLoopChain(chain); - return loopNode; } + + /** + * 将单个参数对象复用基础参数解析逻辑转换为运行时参数。 + * + * @param parameterObject 参数 JSON + * @return 参数对象;输入为空时返回 {@code null} + */ + private Parameter parseParameter(JSONObject parameterObject) { + if (parameterObject == null || parameterObject.isEmpty()) { + return null; + } + JSONArray parameters = new JSONArray(); + parameters.add(parameterObject); + List parsed = getParameters(parameters); + return parsed.isEmpty() ? null : parsed.get(0); + } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeParserTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeParserTest.java new file mode 100644 index 0000000..9979a5d --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeParserTest.java @@ -0,0 +1,95 @@ +package com.easyagents.flow.core.test; + +import com.alibaba.fastjson.JSONObject; +import com.easyagents.flow.core.chain.DataType; +import com.easyagents.flow.core.node.LoopNode; +import com.easyagents.flow.core.parser.impl.LoopNodeParser; +import org.junit.Assert; +import org.junit.Test; + +/** + * 验证循环节点新旧输入结构的解析兼容。 + */ +public class LoopNodeParserTest { + + /** + * 验证新结构可以同时解析循环次数和输入数组。 + */ + @Test + public void shouldParseExplicitLoopInputs() { + JSONObject count = parameter( + "count", "Number", "fixed", null, "3"); + JSONObject items = parameter( + "items", "Array", "ref", + "knowledge.documents", null); + JSONObject loopInputs = new JSONObject(); + loopInputs.put("count", count); + loopInputs.put("items", items); + JSONObject data = new JSONObject(); + data.put("loopInputs", loopInputs); + + LoopNode loopNode = new LoopNodeParser().doParse( + new JSONObject(), data, new JSONObject()); + + Assert.assertNotNull(loopNode.getLoopCount()); + Assert.assertEquals( + DataType.Number, + loopNode.getLoopCount().getDataType()); + Assert.assertEquals( + "3", loopNode.getLoopCount().getValue()); + Assert.assertNotNull(loopNode.getLoopItems()); + Assert.assertEquals( + DataType.Array_Object, + loopNode.getLoopItems().getDataType()); + Assert.assertEquals( + "knowledge.documents", + loopNode.getLoopItems().getRef()); + Assert.assertNull(loopNode.getLoopVar()); + } + + /** + * 验证旧版 loopVars 第一项继续映射为兼容参数。 + */ + @Test + public void shouldParseLegacyLoopVar() { + com.alibaba.fastjson.JSONArray loopVars = + new com.alibaba.fastjson.JSONArray(); + loopVars.add(parameter( + "loopVar", "Number", "fixed", null, "2")); + JSONObject data = new JSONObject(); + data.put("loopVars", loopVars); + + LoopNode loopNode = new LoopNodeParser().doParse( + new JSONObject(), data, new JSONObject()); + + Assert.assertNotNull(loopNode.getLoopVar()); + Assert.assertEquals("2", loopNode.getLoopVar().getValue()); + Assert.assertNull(loopNode.getLoopCount()); + Assert.assertNull(loopNode.getLoopItems()); + } + + /** + * 构造参数 JSON。 + * + * @param name 参数名 + * @param dataType 数据类型 + * @param refType 引用类型 + * @param ref 引用路径 + * @param value 固定值 + * @return 参数 JSON + */ + private static JSONObject parameter( + String name, + String dataType, + String refType, + String ref, + String value) { + JSONObject parameter = new JSONObject(); + parameter.put("name", name); + parameter.put("dataType", dataType); + parameter.put("refType", refType); + parameter.put("ref", ref); + parameter.put("value", value); + return parameter; + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java index 30ea78d..471c83c 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopNodeProgressContextTest.java @@ -237,6 +237,110 @@ public class LoopNodeProgressContextTest { Arrays.asList("1", "2", "3"), primitiveArrayResult.get("result")); } + /** + * 验证次数和数组同时存在时只遍历数组前 N 项。 + */ + @Test + public void shouldLimitArrayLoopByConfiguredCount() { + ChainExecutor executor = createExecutor( + createDualInputDefinition(true, true)); + Map variables = new HashMap<>(); + variables.put("count", 2); + variables.put("items", Arrays.asList("a", "b", "c")); + + Map result = executor.execute( + "loop-dual-input-test", variables); + + Assert.assertEquals( + Arrays.asList("a", "b"), + result.get("result")); + Assert.assertEquals( + Arrays.asList("a", "b", "c"), + result.get("original")); + } + + /** + * 验证次数大于数组长度时只处理现有元素。 + */ + @Test + public void shouldStopAtArrayLengthWhenCountIsLarger() { + ChainExecutor executor = createExecutor( + createDualInputDefinition(true, true)); + Map variables = new HashMap<>(); + variables.put("count", 10); + variables.put("items", Arrays.asList("a", "b", "c")); + + Map result = executor.execute( + "loop-dual-input-short-array-test", variables); + + Assert.assertEquals( + Arrays.asList("a", "b", "c"), + result.get("result")); + } + + /** + * 验证仅配置次数时继续把当前序号作为 loopItem。 + */ + @Test + public void shouldSupportCountOnlyInput() { + ChainExecutor executor = createExecutor( + createDualInputDefinition(true, false)); + + Map result = executor.execute( + "loop-count-only-test", + Collections.singletonMap("count", 3)); + + Assert.assertEquals( + Arrays.asList(0, 1, 2), + result.get("result")); + } + + /** + * 验证配置数组后即使次数合法也拒绝非数组运行值。 + */ + @Test + public void shouldRejectNonArrayItemsWhenCountIsPresent() { + ChainExecutor executor = createExecutor( + createDualInputDefinition(true, true)); + Map variables = new HashMap<>(); + variables.put("count", 2); + variables.put("items", "not-an-array"); + + assertLoopFailure( + executor, + "loop-invalid-items-test", + variables, + RuntimeException.class); + } + + /** + * 验证大数组可通过次数限制只处理前 3 项,且上游数组保持完整。 + */ + @Test + public void shouldAllowLargeArrayWhenCountLimitsPrefix() { + ChainExecutor executor = createExecutor( + createDualInputDefinition(true, true)); + java.util.List items = new java.util.ArrayList<>(); + for (int index = 0; + index < Node.MAX_LOOP_COUNT + 1; + index++) { + items.add(index); + } + Map variables = new HashMap<>(); + variables.put("count", 3); + variables.put("items", items); + + Map result = executor.execute( + "loop-large-prefix-test", variables); + + Assert.assertEquals( + Arrays.asList(0, 1, 2), + result.get("result")); + Assert.assertEquals( + Node.MAX_LOOP_COUNT + 1, + ((java.util.List) result.get("original")).size()); + } + /** * 验证显式循环节点接受 300 次,并拒绝 301 次的数值输入。 */ @@ -371,6 +475,90 @@ public class LoopNodeProgressContextTest { return definition; } + /** + * 创建使用新双输入结构的循环定义。 + * + * @param includeCount 是否配置循环次数 + * @param includeItems 是否配置输入数组 + * @return 工作流定义 + */ + private ChainDefinition createDualInputDefinition( + boolean includeCount, boolean includeItems) { + ChainDefinition definition = new ChainDefinition(); + definition.setId("loop-dual-input-test"); + + StartNode startNode = new StartNode(); + startNode.setId("start"); + java.util.List inputs = + new java.util.ArrayList<>(); + if (includeCount) { + inputs.add(inputParameter("count")); + } + if (includeItems) { + inputs.add(inputParameter("items")); + } + startNode.setParameters(inputs); + + LoopNode loopNode = new LoopNode(); + loopNode.setId("loop"); + loopNode.setName("循环节点"); + if (includeCount) { + Parameter count = new Parameter(); + count.setName("count"); + count.setRef("count"); + count.setRefType(RefType.REF); + count.setDataType(DataType.Number); + loopNode.setLoopCount(count); + } + if (includeItems) { + Parameter items = new Parameter(); + items.setName("items"); + items.setRef("items"); + items.setRefType(RefType.REF); + items.setDataType(DataType.Array); + loopNode.setLoopItems(items); + } + + ItemCollectorNode collectorNode = + new ItemCollectorNode("loop"); + collectorNode.setId("collector"); + collectorNode.setParentId("loop"); + + Parameter loopOutput = new Parameter(); + loopOutput.setName("value"); + loopOutput.setRef("collector.value"); + loopOutput.setRefType(RefType.REF); + loopNode.setOutputDefs( + Collections.singletonList(loopOutput)); + + EndNode endNode = new EndNode(); + endNode.setId("end"); + Parameter result = new Parameter(); + result.setName("result"); + result.setRef("loop.value"); + result.setRefType(RefType.REF); + java.util.List outputs = + new java.util.ArrayList<>(); + outputs.add(result); + if (includeItems) { + Parameter original = new Parameter(); + original.setName("original"); + original.setRef("items"); + original.setRefType(RefType.REF); + outputs.add(original); + } + endNode.setOutputDefs(outputs); + + definition.addNode(startNode); + definition.addNode(loopNode); + definition.addNode(collectorNode); + definition.addNode(endNode); + definition.addEdge(edge("e1", "start", "loop")); + definition.addEdge(edge("e2", "loop", "collector")); + definition.addEdge(edge("e3", "loop", "end")); + return definition; + } + /** * 创建包含两个并行直属循环分支的定义。 * @@ -707,6 +895,32 @@ public class LoopNodeProgressContextTest { } } + /** + * 输出当前 loopItem 的测试节点。 + */ + private static class ItemCollectorNode extends BaseNode { + private final String loopNodeId; + + /** + * 创建当前项收集节点。 + * + * @param loopNodeId 父循环节点 ID + */ + private ItemCollectorNode(String loopNodeId) { + this.loopNodeId = loopNodeId; + } + + /** + * {@inheritDoc} + */ + @Override + public Map execute(Chain chain) { + Object item = chain.getState().resolveValue( + loopNodeId + ".loopItem"); + return Collections.singletonMap("value", item); + } + } + /** * 输出当前循环序号的测试分支。 */ From e74d229de2c235dfb0444f3cb5aa4e47769984c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 30 Jul 2026 17:34:53 +0800 Subject: [PATCH 15/33] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E8=8A=82=E7=82=B9=E8=87=AA=E5=8A=A8=E8=B0=83=E7=94=A8?= =?UTF-8?q?=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将节点输入作为对象参数传入 main,并映射对象返回值 - 保留 _result 兼容并补充异常与回归测试 --- .../code/impl/JavascriptRuntimeEngine.java | 59 ++++++++- .../test/JavascriptRuntimeEngineTest.java | 122 ++++++++++++++++++ 2 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java index 040087e..6496981 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java @@ -15,6 +15,7 @@ */ package com.easyagents.flow.core.code.impl; +import com.alibaba.fastjson.JSONObject; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.NodeState; @@ -25,8 +26,16 @@ import org.graalvm.polyglot.Context; import org.graalvm.polyglot.HostAccess; import org.graalvm.polyglot.Value; +import java.util.Collections; import java.util.Map; +/** + * 基于 GraalVM 的 JavaScript 代码节点执行器。 + * + *

优先保留历史 {@code _result} 输出约定;当脚本未写入 + * {@code _result} 且声明了 {@code main} 函数时,自动传入节点参数并使用 + * {@code main} 返回的对象作为节点输出。

+ */ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { // 使用 Context.Builder 构建上下文,线程安全 @@ -37,6 +46,15 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { .option("js.ecmascript-version", "2021"); // 使用较新的 ECMAScript 版本 + /** + * 执行 JavaScript 代码并返回节点输出。 + * + * @param code 用户代码 + * @param node 当前代码节点 + * @param chain 当前工作流 + * @return 代码节点输出 + * @throws RuntimeException JavaScript 执行失败或 main 返回值不是对象时抛出 + */ @Override public Map execute(String code, CodeNode node, Chain chain) { try (Context context = CONTEXT_BUILDER.build()) { @@ -73,13 +91,48 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { // 执行用户脚本 context.eval("js", code); - Value resultValue = bindings.getMember("_result"); - - return GraalvmToFastJSONUtils.toJSONObject(resultValue); + return resolveResult(context, bindings, parameterValues); } catch (Exception e) { throw new RuntimeException("Polyglot JS 脚本执行失败: " + e.getMessage(), e); } } + /** + * 解析脚本输出,兼容历史 _result 写法并自动调用 main 函数。 + * + * @param context JavaScript 执行上下文 + * @param bindings JavaScript 全局绑定 + * @param parameterValues 当前节点已解析的输入参数 + * @return 代码节点输出 + * @throws IllegalArgumentException main 返回值不是对象时抛出 + */ + private Map resolveResult(Context context, + Value bindings, + Map parameterValues) { + JSONObject legacyResult = GraalvmToFastJSONUtils.toJSONObject( + bindings.getMember("_result")); + if (!legacyResult.isEmpty()) { + return legacyResult; + } + + Value mainFunction = bindings.getMember("main"); + if (mainFunction == null || !mainFunction.canExecute()) { + return legacyResult; + } + + Map mainInput = parameterValues == null + ? Collections.emptyMap() + : parameterValues; + Value mainResult = mainFunction.execute( + JsInteropUtils.wrapJavaValueForJS(context, mainInput)); + Object convertedResult = + GraalvmToFastJSONUtils.toFastJsonValue(mainResult); + if (!(convertedResult instanceof JSONObject)) { + throw new IllegalArgumentException( + "JavaScript main 函数必须返回对象,例如:return { result: value }"); + } + return (JSONObject) convertedResult; + } + } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java new file mode 100644 index 0000000..cf675e4 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java @@ -0,0 +1,122 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.Parameter; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.node.CodeNode; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * JavaScript 代码节点输出约定回归测试。 + */ +public class JavascriptRuntimeEngineTest { + + /** + * 验证运行时自动调用 main,并把节点输入作为对象参数传入。 + */ + @Test + public void shouldInvokeMainAndUseReturnedObject() { + CodeNode node = codeNode( + String.join("\n", + "function main({ data1, data2 }) {", + " return {", + " joined: data1 + data2,", + " totalLength: data1.length + data2.length", + " };", + "}"), + "data1", + "data2"); + Chain chain = chain(Map.of( + "data1", "河北", + "data2", "分行")); + + Map result = node.execute(chain); + + Assert.assertEquals("河北分行", result.get("joined")); + Assert.assertEquals(4L, result.get("totalLength")); + } + + /** + * 验证已有 _result 输出优先,避免自动调用 main 改变历史流程。 + */ + @Test + public void shouldKeepLegacyResultWithoutInvokingMain() { + CodeNode node = codeNode( + String.join("\n", + "_result.answer = 'legacy';", + "function main() {", + " throw new Error('main should not be invoked');", + "}")); + Chain chain = chain(Collections.emptyMap()); + + Map result = node.execute(chain); + + Assert.assertEquals("legacy", result.get("answer")); + } + + /** + * 验证 main 返回非对象时给出明确错误。 + */ + @Test + public void shouldRejectNonObjectMainResult() { + CodeNode node = codeNode( + "function main() { return 'invalid'; }"); + Chain chain = chain(Collections.emptyMap()); + + try { + node.execute(chain); + Assert.fail("main 返回非对象时应执行失败"); + } catch (RuntimeException exception) { + Assert.assertTrue(exception.getMessage().contains( + "JavaScript main 函数必须返回对象")); + } + } + + /** + * 创建 JavaScript 代码节点。 + * + * @param code 用户代码 + * @param parameterNames 输入参数名 + * @return 代码节点 + */ + private CodeNode codeNode(String code, String... parameterNames) { + CodeNode node = new CodeNode(); + node.setId("code-node"); + node.setName("代码节点"); + node.setEngine("js"); + node.setCode(code); + node.setParameters(Arrays.stream(parameterNames) + .map(Parameter::new) + .collect(Collectors.toList())); + return node; + } + + /** + * 创建带初始化状态和输入变量的工作流。 + * + * @param inputs 工作流输入 + * @return 工作流 + */ + private Chain chain(Map inputs) { + Chain chain = new Chain( + new ChainDefinition(), + "javascript-runtime-" + UUID.randomUUID()); + chain.setChainStateRepository( + new InMemoryChainStateRepository()); + chain.setNodeStateRepository( + new InMemoryNodeStateRepository()); + ChainState state = chain.initializeState(); + state.getMemory().putAll(inputs); + return chain; + } +} From fcc36dc699635389cae36a015010853cd3127a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 31 Jul 2026 09:40:38 +0800 Subject: [PATCH 16/33] =?UTF-8?q?feat(XL13):=20=E6=94=AF=E6=8C=81=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E6=B5=81=E5=A4=A7=E6=A8=A1=E5=9E=8B=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 增加文本与思考增量事件及取消终态收口 - 支持图片输入解析并修复死信定义查找空值 - 补充并发与模型流式回归测试 --- .../com/easyagents/flow/core/chain/Chain.java | 16 +- .../flow/core/chain/event/LlmStreamEvent.java | 126 +++++++ .../core/chain/runtime/ChainExecutor.java | 29 +- .../com/easyagents/flow/core/llm/Llm.java | 19 ++ .../easyagents/flow/core/node/LlmNode.java | 23 +- .../test/ChainExecutorConcurrencyTest.java | 98 ++++++ easy-agents-support/pom.xml | 6 +- .../flow/support/provider/EasyAgentsLlm.java | 286 +++++++++++++++- .../support/provider/ImageInputResolver.java | 16 + .../support/provider/EasyAgentsLlmTest.java | 314 ++++++++++++++++++ 10 files changed, 907 insertions(+), 26 deletions(-) create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/LlmStreamEvent.java create mode 100644 easy-agents-support/src/main/java/com/easyagents/flow/support/provider/ImageInputResolver.java create mode 100644 easy-agents-support/src/test/java/com/easyagents/flow/support/provider/EasyAgentsLlmTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java index df5c58e..9fc1fb1 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java @@ -778,7 +778,19 @@ public class Chain { if (state == null) { throw new IllegalStateException("Unable to initialize chain state: " + stateInstanceId); } - return state; + if (StringUtil.hasText(state.getChainDefinitionId()) + || definition == null + || StringUtil.noText(definition.getId())) { + return state; + } + // 定义 ID 必须先于可重放入口触发器持久化,避免恢复时无法定位定义快照。 + return updateStateSafely(current -> { + if (StringUtil.hasText(current.getChainDefinitionId())) { + return null; + } + current.setChainDefinitionId(definition.getId()); + return EnumSet.of(ChainStateField.CHAIN_DEFINITION_ID); + }); } private boolean shouldSkipNode(Node node, String edgeId) { @@ -1613,6 +1625,8 @@ public class Chain { if (changed.get()) { notifyEvent(new ChainStatusChangeEvent( this, ChainStatus.CANCELLED, before.get())); + // 取消属于工作流终态,统一发布结束事件供审计、清理等监听器收口。 + notifyEvent(new ChainEndEvent(this)); } return changed.get(); } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/LlmStreamEvent.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/LlmStreamEvent.java new file mode 100644 index 0000000..bbcdc9f --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/event/LlmStreamEvent.java @@ -0,0 +1,126 @@ +/** + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + *

+ * Licensed under the GNU Lesser General Public License (LGPL) ,Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl-3.0.txt + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.flow.core.chain.event; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.Node; + +/** + * LLM 节点生成文本时发布的增量事件。 + */ +public class LlmStreamEvent extends BaseEvent { + + private final Node node; + private final String streamId; + private final String delta; + private final ContentType contentType; + + /** + * LLM 流式内容类型。 + */ + public enum ContentType { + /** + * 模型正式回答。 + */ + TEXT, + /** + * 模型思考过程。 + */ + REASONING + } + + /** + * 创建 LLM 文本增量事件。 + * + * @param chain 当前工作流 + * @param node 当前 LLM 节点 + * @param streamId 当前节点本次调用的流标识 + * @param delta 本次新增文本 + */ + public LlmStreamEvent(Chain chain, Node node, String streamId, String delta) { + this(chain, node, streamId, delta, ContentType.TEXT); + } + + /** + * 创建指定内容类型的 LLM 增量事件。 + * + * @param chain 当前工作流 + * @param node 当前 LLM 节点 + * @param streamId 当前节点本次调用的流标识 + * @param delta 本次新增内容 + * @param contentType 增量内容类型 + */ + public LlmStreamEvent( + Chain chain, + Node node, + String streamId, + String delta, + ContentType contentType + ) { + super(chain); + this.node = node; + this.streamId = streamId; + this.delta = delta; + this.contentType = contentType == null + ? ContentType.TEXT + : contentType; + } + + /** + * 获取当前 LLM 节点。 + * + * @return 当前节点 + */ + public Node getNode() { + return node; + } + + /** + * 获取当前节点本次调用的流标识。 + * + * @return 流标识 + */ + public String getStreamId() { + return streamId; + } + + /** + * 获取本次新增文本。 + * + * @return 文本增量 + */ + public String getDelta() { + return delta; + } + + /** + * 获取本次增量的内容类型。 + * + * @return 内容类型 + */ + public ContentType getContentType() { + return contentType; + } + + /** + * 判断本次增量是否为模型思考内容。 + * + * @return {@code true} 表示思考内容 + */ + public boolean isReasoning() { + return contentType == ContentType.REASONING; + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java index 130d1a8..c2491e2 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.*; import java.util.concurrent.*; +import java.util.function.Consumer; /** * TinyFlow 最新 ChainExecutor @@ -397,8 +398,29 @@ public class ChainExecutor { } public String executeAsync(String definitionId, Map variables) { + return executeAsync(definitionId, variables, null); + } + + /** + * 异步启动工作流,并在首个节点开始前暴露执行实例 ID。 + * + *

回调用于提前注册流式事件接收器,保证高速工作流不会在调用方拿到执行 ID + * 之前丢失开始事件或首批输出。

+ * + * @param definitionId 工作流定义 ID + * @param variables 工作流输入变量 + * @param beforeStart 启动前回调;可为 {@code null} + * @return 执行实例 ID + */ + public String executeAsync( + String definitionId, + Map variables, + Consumer beforeStart) { Chain chain = createChain(definitionId); try { + if (beforeStart != null) { + beforeStart.accept(chain.getStateInstanceId()); + } chain.start(variables); return chain.getStateInstanceId(); } catch (RuntimeException | Error error) { @@ -962,9 +984,12 @@ public class ChainExecutor { return definition; } ChainDefinition loaded = definitionSnapshotRepository.load(stateInstanceId); - if (loaded == null) { + String definitionId = state.getChainDefinitionId(); + if (loaded == null + && definitionId != null + && !definitionId.isBlank()) { // 兼容升级前已经启动、尚未持久化定义快照的实例。 - loaded = definitionRepository.getChainDefinitionById(state.getChainDefinitionId()); + loaded = definitionRepository.getChainDefinitionById(definitionId); } if (loaded == null) { return null; diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java index d681115..1bcf699 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/llm/Llm.java @@ -50,6 +50,7 @@ public interface Llm { private String message; private String systemMessage; private List images; + private List imageInputs; public String getMessage() { return message; @@ -74,6 +75,24 @@ public interface Llm { public void setImages(List images) { this.images = images; } + + /** + * 获取尚未转换为模型图片 URL 的原始图片输入。 + * + * @return 原始图片输入列表 + */ + public List getImageInputs() { + return imageInputs; + } + + /** + * 设置原始图片输入,供模型调用前按运行环境解析。 + * + * @param imageInputs 原始图片输入列表 + */ + public void setImageInputs(List imageInputs) { + this.imageInputs = imageInputs; + } } /** diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java index e62e31b..f2e3e0f 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LlmNode.java @@ -23,7 +23,6 @@ import com.easyagents.flow.core.llm.Llm; import com.easyagents.flow.core.llm.LlmManager; import com.easyagents.flow.core.util.*; -import java.io.File; import java.util.*; public class LlmNode extends BaseNode { @@ -123,17 +122,21 @@ public class LlmNode extends BaseNode { Map filesMap = chainState.resolveParameters( this, images); - List imagesUrls = new ArrayList<>(); - filesMap.forEach((s, o) -> { - if (o instanceof String) { - imagesUrls.add((String) o); - } else if (o instanceof File) { - byte[] bytes = IOUtil.readBytes((File) o); - String base64 = Base64.getEncoder().encodeToString(bytes); - imagesUrls.add(base64); + List imageInputs = new ArrayList<>(filesMap.size()); + filesMap.forEach((name, value) -> { + if (value == null) { + return; } + if (!(value instanceof String) + && !(value instanceof java.io.File) + && !(value instanceof Map)) { + throw new IllegalArgumentException( + "Unsupported image input for parameter '" + name + "': " + + value.getClass().getName()); + } + imageInputs.add(value); }); - messageInfo.setImages(imagesUrls); + messageInfo.setImageInputs(imageInputs); } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java index f1547eb..b9d402d 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java @@ -20,6 +20,7 @@ import com.easyagents.flow.core.chain.ChainDefinition; import com.easyagents.flow.core.chain.ChainStatus; import com.easyagents.flow.core.chain.Edge; import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.event.ChainEndEvent; import com.easyagents.flow.core.chain.repository.ChainDefinitionSnapshotRepository; import com.easyagents.flow.core.chain.repository.ChainStateField; import com.easyagents.flow.core.chain.repository.ChainStateRepository; @@ -28,6 +29,7 @@ import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; import com.easyagents.flow.core.chain.runtime.ChainExecutor; import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; +import com.easyagents.flow.core.chain.runtime.Trigger; import com.easyagents.flow.core.chain.runtime.TriggerScheduler; import com.easyagents.flow.core.node.EndNode; import com.easyagents.flow.core.node.BaseNode; @@ -36,6 +38,7 @@ import org.junit.Assert; import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -49,12 +52,100 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * {@link ChainExecutor} 并发同步执行测试。 */ public class ChainExecutorConcurrencyTest { + /** + * 验证实例初始化会在入口触发器创建前持久化工作流定义 ID。 + * + * @throws Exception 执行器初始化或异步启动失败时抛出 + */ + @Test + public void shouldPersistDefinitionIdBeforeStartingWorkflow() throws Exception { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newFixedThreadPool(2); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + ChainDefinition definition = createDefinition(); + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + ChainExecutor chainExecutor = new ChainExecutor( + id -> definition, + stateRepository, + new InMemoryNodeStateRepository(), + triggerScheduler); + AtomicReference persistedDefinitionId = new AtomicReference<>(); + + try { + chainExecutor.executeAsync( + definition.getId(), + Collections.emptyMap(), + executeId -> persistedDefinitionId.set( + stateRepository.load(executeId).getChainDefinitionId()) + ); + + Assert.assertEquals( + definition.getId(), + persistedDefinitionId.get()); + } finally { + triggerScheduler.shutdown(); + } + } + + /** + * 验证历史异常实例缺少定义 ID 时,死信触发器仍能收敛为失败终态。 + * + * @throws Exception 反射调用死信收口逻辑失败时抛出 + */ + @Test + public void shouldFailDeadLetteredWorkflowWithoutDefinitionId() + throws Exception { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newFixedThreadPool(2); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 1000L); + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + AtomicInteger definitionLoadCount = new AtomicInteger(); + ChainExecutor chainExecutor = new ChainExecutor( + id -> { + definitionLoadCount.incrementAndGet(); + return null; + }, + stateRepository, + new InMemoryNodeStateRepository(), + triggerScheduler); + String instanceId = "dead-letter-missing-definition"; + stateRepository.create(instanceId); + Trigger trigger = new Trigger(); + trigger.setId("dead-letter-trigger"); + trigger.setStateInstanceId(instanceId); + Method failDeadLetteredTrigger = ChainExecutor.class.getDeclaredMethod( + "failDeadLetteredTrigger", + Trigger.class, + Throwable.class); + failDeadLetteredTrigger.setAccessible(true); + + try { + boolean finalized = (boolean) failDeadLetteredTrigger.invoke( + chainExecutor, + trigger, + new IllegalStateException("delivery attempts exhausted")); + + Assert.assertTrue(finalized); + Assert.assertEquals( + ChainStatus.FAILED, + stateRepository.load(instanceId).getStatus()); + Assert.assertEquals(0, definitionLoadCount.get()); + } finally { + triggerScheduler.shutdown(); + } + } + /** * 验证多个同步调用可以通过实例 ID 独立接收执行结果。 * @@ -143,6 +234,7 @@ public class ChainExecutorConcurrencyTest { CountDownLatch ioStarted = new CountDownLatch(1); CountDownLatch allowIoCompletion = new CountDownLatch(1); AtomicInteger downstreamExecutions = new AtomicInteger(); + AtomicInteger chainEndEvents = new AtomicInteger(); ChainDefinition definition = createCancellationDefinition( ioStarted, allowIoCompletion, downstreamExecutions); ChainExecutor chainExecutor = new ChainExecutor( @@ -150,6 +242,11 @@ public class ChainExecutorConcurrencyTest { chainStateRepository, new InMemoryNodeStateRepository(), triggerScheduler); + chainExecutor.addEventListener((event, chain) -> { + if (event instanceof ChainEndEvent) { + chainEndEvents.incrementAndGet(); + } + }); try { String instanceId = chainExecutor.executeAsync( @@ -163,6 +260,7 @@ public class ChainExecutorConcurrencyTest { Assert.assertEquals( ChainStatus.CANCELLED, chainStateRepository.load(instanceId).getStatus()); + Assert.assertEquals(1, chainEndEvents.get()); } finally { allowIoCompletion.countDown(); triggerScheduler.shutdown(); diff --git a/easy-agents-support/pom.xml b/easy-agents-support/pom.xml index b25b315..0dfe000 100644 --- a/easy-agents-support/pom.xml +++ b/easy-agents-support/pom.xml @@ -28,7 +28,11 @@ easy-agents-core - + + junit + junit + test + diff --git a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java index 9ab2422..d09d295 100644 --- a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java +++ b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/EasyAgentsLlm.java @@ -2,46 +2,240 @@ package com.easyagents.flow.support.provider; import com.easyagents.core.message.AiMessage; import com.easyagents.core.message.SystemMessage; +import com.easyagents.core.model.chat.BaseChatModel; import com.easyagents.core.model.chat.ChatModel; +import com.easyagents.core.model.chat.StreamResponseListener; +import com.easyagents.core.model.client.StreamContext; import com.easyagents.core.model.chat.response.AiMessageResponse; import com.easyagents.core.prompt.SimplePrompt; +import com.easyagents.core.util.ImageUtil; import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent; +import com.easyagents.flow.core.chain.event.LlmStreamEvent; +import com.easyagents.flow.core.chain.listener.ChainEventListener; import com.easyagents.flow.core.llm.Llm; import com.easyagents.flow.core.node.LlmNode; import com.easyagents.flow.core.util.StringUtil; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +/** + * 基于 Easy-Agents 聊天模型实现工作流 LLM 调用。 + */ public class EasyAgentsLlm implements Llm { private ChatModel chatModel; + private ImageInputResolver imageInputResolver; + /** + * 获取聊天模型。 + * + * @return 聊天模型 + */ public ChatModel getChatModel() { return chatModel; } + /** + * 设置聊天模型。 + * + * @param chatModel 聊天模型 + */ public void setChatModel(ChatModel chatModel) { this.chatModel = chatModel; } + /** + * 获取图片输入解析器。 + * + * @return 图片输入解析器 + */ + public ImageInputResolver getImageInputResolver() { + return imageInputResolver; + } + + /** + * 设置图片输入解析器。 + * + * @param imageInputResolver 图片输入解析器 + */ + public void setImageInputResolver(ImageInputResolver imageInputResolver) { + this.imageInputResolver = imageInputResolver; + } + + /** + * 调用聊天模型并返回文本结果。 + * + * @param messageInfo 消息信息 + * @param options 模型参数 + * @param llmNode 当前 LLM 节点 + * @param chain 工作流链 + * @return 模型文本结果 + */ @Override public String chat(MessageInfo messageInfo, ChatOptions options, LlmNode llmNode, Chain chain) { + SimplePrompt prompt = buildPrompt(messageInfo); + com.easyagents.core.model.chat.ChatOptions chatOptions = buildChatOptions(options); + CountDownLatch completion = new CountDownLatch(1); + AtomicReference streamContext = new AtomicReference<>(); + AtomicReference result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + String streamId = UUID.randomUUID().toString(); + + ChainEventListener cancellationListener = (event, eventChain) -> { + if (!(event instanceof ChainStatusChangeEvent statusEvent) + || statusEvent.getStatus() != ChainStatus.CANCELLED + || !chain.getStateInstanceId().equals(eventChain.getStateInstanceId())) { + return; + } + StreamContext context = streamContext.get(); + if (context != null) { + context.getClient().stop(); + } + }; + chain.getEventManager().addEventListener( + ChainStatusChangeEvent.class, cancellationListener); + + try { + chatModel.chatStream(prompt, new StreamResponseListener() { + /** + * 记录流客户端,供工作流取消时立即关闭模型连接。 + * + * @param context 流上下文 + */ + @Override + public void onStart(StreamContext context) { + streamContext.set(context); + } + + /** + * 将模型文本增量转发为工作流事件。 + * + * @param context 流上下文 + * @param response 本次模型响应 + */ + @Override + public void onMessage(StreamContext context, AiMessageResponse response) { + AiMessage message = response == null ? null : response.getMessage(); + if (message == null) { + return; + } + String reasoningDelta = message.getReasoningContent(); + if (StringUtil.hasText(reasoningDelta)) { + chain.notifyEvent(new LlmStreamEvent( + chain, + llmNode, + streamId, + reasoningDelta, + LlmStreamEvent.ContentType.REASONING + )); + } + String textDelta = message.getContent(); + if (StringUtil.hasText(textDelta)) { + chain.notifyEvent(new LlmStreamEvent( + chain, + llmNode, + streamId, + textDelta, + LlmStreamEvent.ContentType.TEXT + )); + } + } + + /** + * 收集完整响应并结束同步节点等待。 + * + * @param context 流上下文 + */ + @Override + public void onStop(StreamContext context) { + try { + if (failure.get() == null) { + AiMessage message = context.getFullMessage(); + if (message == null || StringUtil.noText(message.getFullContent())) { + failure.compareAndSet( + null, + new IllegalStateException( + "EasyAgentsLlm can not get aiMessage!")); + } else { + result.set(message.getFullContent()); + } + } + } finally { + completion.countDown(); + } + } + + /** + * 记录模型流异常并结束等待。 + * + * @param context 流上下文 + * @param throwable 模型异常 + */ + @Override + public void onFailure(StreamContext context, Throwable throwable) { + failure.compareAndSet( + null, + throwable == null + ? new IllegalStateException("EasyAgentsLlm stream failed") + : throwable); + completion.countDown(); + } + }, chatOptions); + awaitCompletion(completion, streamContext); + } finally { + chain.getEventManager().removeEventListener( + ChainStatusChangeEvent.class, cancellationListener); + } + + Throwable throwable = failure.get(); + if (throwable != null) { + throw new RuntimeException("EasyAgentsLlm stream failed", throwable); + } + if (StringUtil.noText(result.get())) { + throw new RuntimeException("EasyAgentsLlm can not get response!"); + } + return result.get(); + } + + /** + * 构建模型提示词,并解析图片输入。 + * + * @param messageInfo 消息信息 + * @return 模型提示词 + */ + private SimplePrompt buildPrompt(MessageInfo messageInfo) { SimplePrompt prompt = new SimplePrompt(messageInfo.getMessage()); - // 系统提示词 if (StringUtil.hasText(messageInfo.getSystemMessage())) { prompt.setSystemMessage(SystemMessage.of(messageInfo.getSystemMessage())); } - // 图片 - List images = messageInfo.getImages(); + List images = resolveImages(messageInfo); if (images != null && !images.isEmpty()) { + assertImageSupported(); for (String image : images) { prompt.addImageUrl(image); } } + return prompt; + } + /** + * 构建 Easy-Agents 模型参数。 + * + * @param options 工作流模型参数 + * @return Easy-Agents 模型参数 + */ + private com.easyagents.core.model.chat.ChatOptions buildChatOptions(ChatOptions options) { com.easyagents.core.model.chat.ChatOptions chatOptions = new com.easyagents.core.model.chat.ChatOptions(); chatOptions.setSeed(options.getSeed()); chatOptions.setTemperature(options.getTemperature()); @@ -49,21 +243,89 @@ public class EasyAgentsLlm implements Llm { chatOptions.setTopK(options.getTopK()); chatOptions.setMaxTokens(options.getMaxTokens()); chatOptions.setStop(options.getStop()); + return chatOptions; + } - AiMessageResponse response = chatModel.chat(prompt, chatOptions); - if (response == null) { - throw new RuntimeException("EasyAgentsLlm can not get response!"); + /** + * 等待异步模型流结束。 + * + * @param completion 完成信号 + * @param streamContext 当前模型流上下文 + */ + private void awaitCompletion( + CountDownLatch completion, + AtomicReference streamContext) { + try { + completion.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + StreamContext context = streamContext.get(); + if (context != null) { + context.getClient().stop(); + } + throw new RuntimeException("EasyAgentsLlm stream interrupted", exception); + } + } + + /** + * 将原始图片输入解析为模型图片 URL。 + * + * @param messageInfo 消息信息 + * @return 已解析图片 URL 列表 + */ + List resolveImages(MessageInfo messageInfo) { + if (messageInfo == null) { + return Collections.emptyList(); + } + List inputs = messageInfo.getImageInputs(); + if (inputs == null || inputs.isEmpty()) { + inputs = messageInfo.getImages(); + } + if (inputs == null || inputs.isEmpty()) { + return Collections.emptyList(); } - if (response.isError()) { - throw new RuntimeException("EasyAgentsLlm error: " + response.getErrorMessage()); + List resolvedImages = new ArrayList<>(inputs.size()); + for (Object input : inputs) { + if (input == null) { + continue; + } + String resolvedImage = resolveImage(input); + if (StringUtil.noText(resolvedImage)) { + throw new IllegalArgumentException("Resolved image input must not be blank"); + } + resolvedImages.add(resolvedImage); } + return resolvedImages; + } - AiMessage aiMessage = response.getMessage(); - if (aiMessage != null) { - return aiMessage.getTextContent(); + /** + * 解析单个图片输入。 + * + * @param imageInput 原始图片输入 + * @return 图片 URL 或 Data URI + */ + private String resolveImage(Object imageInput) { + if (imageInputResolver != null) { + return imageInputResolver.resolve(imageInput); } + if (imageInput instanceof String value) { + return value; + } + if (imageInput instanceof File file) { + return ImageUtil.imageFileToDataUri(file); + } + throw new IllegalArgumentException( + "Unsupported image input type: " + imageInput.getClass().getName()); + } - throw new RuntimeException("EasyAgentsLlm can not get aiMessage!"); + /** + * 校验当前聊天模型是否明确支持图片。 + */ + private void assertImageSupported() { + if (chatModel instanceof BaseChatModel baseChatModel + && Boolean.FALSE.equals(baseChatModel.getConfig().getSupportImage())) { + throw new IllegalArgumentException("当前模型不支持图片输入,请选择支持视觉能力的模型"); + } } } diff --git a/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/ImageInputResolver.java b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/ImageInputResolver.java new file mode 100644 index 0000000..9bdf14d --- /dev/null +++ b/easy-agents-support/src/main/java/com/easyagents/flow/support/provider/ImageInputResolver.java @@ -0,0 +1,16 @@ +package com.easyagents.flow.support.provider; + +/** + * 将工作流运行态图片输入解析为模型可消费的图片 URL 或 Data URI。 + */ +@FunctionalInterface +public interface ImageInputResolver { + + /** + * 解析单个图片输入。 + * + * @param imageInput 原始图片输入 + * @return 图片 URL 或带 MIME 的 Data URI + */ + String resolve(Object imageInput); +} diff --git a/easy-agents-support/src/test/java/com/easyagents/flow/support/provider/EasyAgentsLlmTest.java b/easy-agents-support/src/test/java/com/easyagents/flow/support/provider/EasyAgentsLlmTest.java new file mode 100644 index 0000000..ea51d0e --- /dev/null +++ b/easy-agents-support/src/test/java/com/easyagents/flow/support/provider/EasyAgentsLlmTest.java @@ -0,0 +1,314 @@ +package com.easyagents.flow.support.provider; + +import com.easyagents.core.model.chat.BaseChatModel; +import com.easyagents.core.model.chat.ChatConfig; +import com.easyagents.core.model.chat.ChatModel; +import com.easyagents.core.model.chat.ChatOptions; +import com.easyagents.core.model.chat.StreamResponseListener; +import com.easyagents.core.model.chat.response.AiMessageResponse; +import com.easyagents.core.model.client.StreamClient; +import com.easyagents.core.model.client.StreamContext; +import com.easyagents.core.prompt.Prompt; +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.EventManager; +import com.easyagents.flow.core.chain.event.LlmStreamEvent; +import com.easyagents.flow.core.llm.Llm; +import com.easyagents.flow.core.node.LlmNode; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * 工作流 LLM 图片输入解析测试。 + */ +public class EasyAgentsLlmTest { + + /** + * 验证结构化图片对象通过注入的解析器转换。 + */ + @Test + public void shouldResolveRawImageInput() { + EasyAgentsLlm llm = new EasyAgentsLlm(); + llm.setImageInputResolver(input -> { + Assert.assertTrue(input instanceof Map); + return "data:image/png;base64,AQID"; + }); + Llm.MessageInfo messageInfo = new Llm.MessageInfo(); + messageInfo.setImageInputs(List.of(Map.of( + "sourceType", "upload", + "fileName", "image.png", + "filePath", "workflow/image.png"))); + + Assert.assertEquals( + List.of("data:image/png;base64,AQID"), + llm.resolveImages(messageInfo)); + } + + /** + * 验证旧版图片字符串仍然可直接传递。 + */ + @Test + public void shouldKeepLegacyImageString() { + EasyAgentsLlm llm = new EasyAgentsLlm(); + Llm.MessageInfo messageInfo = new Llm.MessageInfo(); + messageInfo.setImages(List.of("https://example.com/image.png")); + + Assert.assertEquals( + List.of("https://example.com/image.png"), + llm.resolveImages(messageInfo)); + } + + /** + * 验证 File 图片会补全 MIME 和 Data URI 前缀。 + * + * @throws Exception 临时文件写入失败 + */ + @Test + public void shouldConvertFileToCompleteDataUri() throws Exception { + Path image = Files.createTempFile("easy-agents-image-", ".png"); + try { + Files.write(image, new byte[]{1, 2, 3}); + EasyAgentsLlm llm = new EasyAgentsLlm(); + Llm.MessageInfo messageInfo = new Llm.MessageInfo(); + messageInfo.setImageInputs(List.of(image.toFile())); + + Assert.assertEquals( + List.of("data:image/png;base64,AQID"), + llm.resolveImages(messageInfo)); + } finally { + Files.deleteIfExists(image); + } + } + + /** + * 验证明确不支持图片的模型会在发送请求前失败。 + */ + @Test + public void shouldRejectImageForUnsupportedModel() { + ChatConfig config = new ChatConfig(); + config.setSupportImage(Boolean.FALSE); + EasyAgentsLlm llm = new EasyAgentsLlm(); + llm.setChatModel(new BaseChatModel<>(config) { + }); + Llm.MessageInfo messageInfo = new Llm.MessageInfo(); + messageInfo.setImages(List.of("data:image/png;base64,AQID")); + + try { + llm.chat(messageInfo, new Llm.ChatOptions(), null, null); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException exception) { + Assert.assertEquals( + "当前模型不支持图片输入,请选择支持视觉能力的模型", + exception.getMessage()); + } + } + + /** + * 验证重复执行同一 LLM 节点时,每次调用拥有独立流标识且增量不会被覆盖。 + */ + @Test + public void shouldEmitIndependentStreamForRepeatedNodeInvocation() { + EasyAgentsLlm llm = new EasyAgentsLlm(); + llm.setChatModel(new DeterministicStreamChatModel()); + + Chain chain = new Chain(new ChainDefinition(), "stream-test"); + EventManager eventManager = new EventManager(); + chain.setEventManager(eventManager); + List events = new ArrayList<>(); + eventManager.addEventListener( + LlmStreamEvent.class, + (event, currentChain) -> events.add((LlmStreamEvent) event)); + + LlmNode node = new LlmNode(); + node.setId("llm-1"); + Llm.MessageInfo messageInfo = new Llm.MessageInfo(); + messageInfo.setMessage("请回答"); + + Assert.assertEquals( + "你好", + llm.chat(messageInfo, new Llm.ChatOptions(), node, chain)); + Assert.assertEquals( + "你好", + llm.chat(messageInfo, new Llm.ChatOptions(), node, chain)); + + Assert.assertEquals(4, events.size()); + Assert.assertEquals(List.of("你", "好", "你", "好"), events.stream() + .map(LlmStreamEvent::getDelta) + .toList()); + Assert.assertTrue(events.stream().noneMatch(LlmStreamEvent::isReasoning)); + Assert.assertEquals(events.get(0).getStreamId(), events.get(1).getStreamId()); + Assert.assertEquals(events.get(2).getStreamId(), events.get(3).getStreamId()); + Assert.assertNotEquals(events.get(0).getStreamId(), events.get(2).getStreamId()); + } + + /** + * 验证模型思考与正式回答使用同一流标识并按内容类型分别发布。 + */ + @Test + public void shouldEmitReasoningAndTextDeltasSeparately() { + EasyAgentsLlm llm = new EasyAgentsLlm(); + llm.setChatModel(new ReasoningStreamChatModel()); + + Chain chain = new Chain(new ChainDefinition(), "reasoning-stream-test"); + EventManager eventManager = new EventManager(); + chain.setEventManager(eventManager); + List events = new ArrayList<>(); + eventManager.addEventListener( + LlmStreamEvent.class, + (event, currentChain) -> events.add((LlmStreamEvent) event)); + + LlmNode node = new LlmNode(); + node.setId("llm-reasoning"); + Llm.MessageInfo messageInfo = new Llm.MessageInfo(); + messageInfo.setMessage("请回答"); + + Assert.assertEquals( + "答案", + llm.chat(messageInfo, new Llm.ChatOptions(), node, chain)); + Assert.assertEquals(4, events.size()); + Assert.assertEquals( + List.of("先", "想", "答", "案"), + events.stream().map(LlmStreamEvent::getDelta).toList()); + Assert.assertEquals( + List.of( + LlmStreamEvent.ContentType.REASONING, + LlmStreamEvent.ContentType.REASONING, + LlmStreamEvent.ContentType.TEXT, + LlmStreamEvent.ContentType.TEXT + ), + events.stream().map(LlmStreamEvent::getContentType).toList()); + Assert.assertEquals( + 1, + events.stream().map(LlmStreamEvent::getStreamId).distinct().count()); + } + + /** + * 固定输出两个文本增量的测试聊天模型。 + */ + private static final class DeterministicStreamChatModel implements ChatModel { + + /** + * 同步聊天接口不参与本测试。 + * + * @param prompt 提示词 + * @param options 模型参数 + * @return 无 + */ + @Override + public AiMessageResponse chat(Prompt prompt, ChatOptions options) { + throw new UnsupportedOperationException("sync chat is not used"); + } + + /** + * 连续发送两个增量及完整消息。 + * + * @param prompt 提示词 + * @param listener 流监听器 + * @param options 模型参数 + */ + @Override + public void chatStream( + Prompt prompt, + StreamResponseListener listener, + ChatOptions options) { + StreamContext context = new StreamContext(this, null, new NoopStreamClient()); + listener.onStart(context); + for (String content : List.of("你", "好")) { + com.easyagents.core.message.AiMessage delta = + new com.easyagents.core.message.AiMessage(); + delta.setContent(content); + listener.onMessage(context, new AiMessageResponse(null, content, delta)); + } + context.setFullMessage(new com.easyagents.core.message.AiMessage("你好")); + listener.onStop(context); + } + } + + /** + * 固定输出思考与回答增量的测试聊天模型。 + */ + private static final class ReasoningStreamChatModel implements ChatModel { + + /** + * 同步聊天接口不参与本测试。 + * + * @param prompt 提示词 + * @param options 模型参数 + * @return 无 + */ + @Override + public AiMessageResponse chat(Prompt prompt, ChatOptions options) { + throw new UnsupportedOperationException("sync chat is not used"); + } + + /** + * 依次发送思考增量、回答增量及完整消息。 + * + * @param prompt 提示词 + * @param listener 流监听器 + * @param options 模型参数 + */ + @Override + public void chatStream( + Prompt prompt, + StreamResponseListener listener, + ChatOptions options) { + StreamContext context = new StreamContext(this, null, new NoopStreamClient()); + listener.onStart(context); + for (String reasoning : List.of("先", "想")) { + com.easyagents.core.message.AiMessage delta = + new com.easyagents.core.message.AiMessage(); + delta.setReasoningContent(reasoning); + listener.onMessage(context, new AiMessageResponse(null, reasoning, delta)); + } + for (String content : List.of("答", "案")) { + com.easyagents.core.message.AiMessage delta = + new com.easyagents.core.message.AiMessage(); + delta.setContent(content); + listener.onMessage(context, new AiMessageResponse(null, content, delta)); + } + com.easyagents.core.message.AiMessage fullMessage = + new com.easyagents.core.message.AiMessage("答案"); + fullMessage.setFullReasoningContent("先想"); + context.setFullMessage(fullMessage); + listener.onStop(context); + } + } + + /** + * 测试使用的空流客户端。 + */ + private static final class NoopStreamClient implements StreamClient { + + /** + * 测试模型直接分发事件,无需启动网络请求。 + * + * @param url 请求地址 + * @param headers 请求头 + * @param payload 请求体 + * @param listener 客户端监听器 + * @param config 模型配置 + */ + @Override + public void start( + String url, + Map headers, + String payload, + com.easyagents.core.model.client.StreamClientListener listener, + ChatConfig config) { + } + + /** + * 测试客户端没有需要关闭的网络资源。 + */ + @Override + public void stop() { + } + } +} From 4af2d7cd342e29336bc87917a8d8546153b0bdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 31 Jul 2026 14:45:11 +0800 Subject: [PATCH 17/33] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E6=95=B0=E7=BB=84=E8=BE=93=E5=87=BA=E6=89=81=E5=B9=B3?= =?UTF-8?q?=E8=81=9A=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 按输出参数保存并解析一层扁平聚合策略 - 覆盖解析、顺序聚合与异常类型校验 --- .../easyagents/flow/core/chain/Parameter.java | 23 ++++ .../chain/repository/LoopResultReference.java | 42 ++++++ .../repository/LoopResultRepository.java | 124 +++++++++++++++++- .../easyagents/flow/core/node/LoopNode.java | 23 +++- .../flow/core/parser/BaseNodeParser.java | 2 + .../flow/core/test/LoopNodeParserTest.java | 30 +++++ .../test/LoopResultReferenceResolverTest.java | 93 +++++++++++++ 7 files changed, 332 insertions(+), 5 deletions(-) diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java index 97f4bc9..379629f 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Parameter.java @@ -38,6 +38,10 @@ public class Parameter implements Serializable, Cloneable { protected String value; protected boolean required; protected String defaultValue; + /** + * 是否在循环节点完成后将各轮数组输出合并一层。 + */ + protected boolean flattenAggregation; protected List children; /** @@ -167,6 +171,24 @@ public class Parameter implements Serializable, Cloneable { this.defaultValue = defaultValue; } + /** + * 判断循环输出是否启用一层扁平聚合。 + * + * @return 启用时返回 {@code true} + */ + public boolean isFlattenAggregation() { + return flattenAggregation; + } + + /** + * 设置循环输出的一层扁平聚合开关。 + * + * @param flattenAggregation 是否启用 + */ + public void setFlattenAggregation(boolean flattenAggregation) { + this.flattenAggregation = flattenAggregation; + } + public boolean isRequired() { return required; } @@ -273,6 +295,7 @@ public class Parameter implements Serializable, Cloneable { ", value='" + value + '\'' + ", required=" + required + ", defaultValue='" + defaultValue + '\'' + + ", flattenAggregation=" + flattenAggregation + ", children=" + children + ", enums=" + enums + ", formType='" + formType + '\'' + diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java index cd1a740..9a2ea51 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultReference.java @@ -15,6 +15,7 @@ public final class LoopResultReference implements Serializable { private final String resultId; private final int iterationCount; private final String outputName; + private final boolean flattenAggregation; /** * 创建循环结果引用。 @@ -24,23 +25,64 @@ public final class LoopResultReference implements Serializable { * @param outputName 输出名称 */ public LoopResultReference(String resultId, int iterationCount, String outputName) { + this(resultId, iterationCount, outputName, false); + } + + /** + * 创建带聚合策略的循环结果引用。 + * + * @param resultId 循环结果 ID + * @param iterationCount 迭代次数 + * @param outputName 输出名称 + * @param flattenAggregation 是否将各轮数组合并一层 + */ + public LoopResultReference( + String resultId, + int iterationCount, + String outputName, + boolean flattenAggregation) { this.resultId = Objects.requireNonNull(resultId, "resultId must not be null"); this.iterationCount = iterationCount; this.outputName = Objects.requireNonNull(outputName, "outputName must not be null"); + this.flattenAggregation = flattenAggregation; } + /** + * 获取循环结果 ID。 + * + * @return 循环结果 ID + */ public String getResultId() { return resultId; } + /** + * 获取累计迭代次数。 + * + * @return 累计迭代次数 + */ public int getIterationCount() { return iterationCount; } + /** + * 获取输出名称。 + * + * @return 输出名称 + */ public String getOutputName() { return outputName; } + /** + * 判断是否将各轮数组输出合并一层。 + * + * @return 启用时返回 {@code true} + */ + public boolean isFlattenAggregation() { + return flattenAggregation; + } + /** * 获取跨异步审计边界使用的稳定引用类型。 * diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java index 55cd40c..a9776f2 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/repository/LoopResultRepository.java @@ -21,6 +21,7 @@ import java.util.Iterator; import java.util.List; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; /** @@ -295,12 +296,38 @@ public interface LoopResultRepository { */ default Map references( String resultId, int iterationCount, List outputNames) { + return references( + resultId, + iterationCount, + outputNames, + java.util.Collections.emptySet()); + } + + /** + * 为每个循环输出创建带聚合策略的轻量引用。 + * + * @param resultId 循环结果 ID + * @param iterationCount 已累计迭代数 + * @param outputNames 输出名称 + * @param flattenedOutputNames 启用一层扁平聚合的输出名称 + * @return 输出名称到轻量引用的映射 + */ + default Map references( + String resultId, + int iterationCount, + List outputNames, + Set flattenedOutputNames) { Map references = new LinkedHashMap<>(); if (outputNames != null) { for (String outputName : outputNames) { references.put( outputName, - new LoopResultReference(resultId, iterationCount, outputName)); + new LoopResultReference( + resultId, + iterationCount, + outputName, + flattenedOutputNames != null + && flattenedOutputNames.contains(outputName))); } } return references; @@ -310,14 +337,15 @@ public interface LoopResultRepository { * 解析单个循环输出引用。 * * @param reference 循环输出引用 - * @return 与旧实现相同的累计列表 + * @return 未启用时返回按轮次累计的列表,启用时返回只扁平一层的数组 */ default Object resolve(LoopResultReference reference) { - return load( + Object output = load( reference.getResultId(), reference.getIterationCount(), List.of(reference.getOutputName())) .get(reference.getOutputName()); + return ReferenceResolver.applyAggregation(reference, output); } /** @@ -420,7 +448,10 @@ public interface LoopResultRepository { LoopResultReference reference = (LoopResultReference) value; Map outputs = loaded.get(new GroupKey( reference.getResultId(), reference.getIterationCount())); - return outputs == null ? null : outputs.get(reference.getOutputName()); + Object output = outputs == null + ? null + : outputs.get(reference.getOutputName()); + return applyAggregation(reference, output); } if (value instanceof LoopInputReference) { return loadedInputs.get( @@ -451,6 +482,91 @@ public interface LoopResultRepository { return value; } + /** + * 根据循环结果引用对累计输出执行一次线性聚合。 + * + * @param reference 循环结果引用 + * @param output 按轮次累计的原始输出 + * @return 原始输出或只扁平一层后的数组 + * @throws IllegalStateException 启用扁平聚合但某轮值不是数组 + */ + private static Object applyAggregation( + LoopResultReference reference, Object output) { + if (!reference.isFlattenAggregation()) { + return output; + } + if (!(output instanceof List)) { + throw invalidFlattenValue(reference, -1, output); + } + + List iterationValues = (List) output; + int flattenedSize = 0; + int iterationIndex = 0; + // 使用顺序迭代兼容链表,避免按索引读取退化为 O(n²)。 + for (Object iterationValue : iterationValues) { + int currentSize; + if (iterationValue instanceof List) { + currentSize = ((List) iterationValue).size(); + } else if (iterationValue != null + && iterationValue.getClass().isArray()) { + currentSize = java.lang.reflect.Array.getLength(iterationValue); + } else { + throw invalidFlattenValue( + reference, iterationIndex, iterationValue); + } + try { + flattenedSize = Math.addExact(flattenedSize, currentSize); + } catch (ArithmeticException exception) { + throw new IllegalStateException( + "Loop output '" + reference.getOutputName() + + "' is too large to flatten", + exception); + } + iterationIndex++; + } + + // 先统计容量再顺序追加,避免大数组扩容复制,整体复杂度保持 O(n)。 + java.util.ArrayList flattened = + new java.util.ArrayList<>(flattenedSize); + for (Object iterationValue : iterationValues) { + if (iterationValue instanceof List) { + flattened.addAll((List) iterationValue); + continue; + } + int length = java.lang.reflect.Array.getLength(iterationValue); + for (int index = 0; index < length; index++) { + flattened.add( + java.lang.reflect.Array.get(iterationValue, index)); + } + } + return flattened; + } + + /** + * 构造扁平聚合类型错误。 + * + * @param reference 循环结果引用 + * @param iterationIndex 错误轮次;负数表示累计输出结构错误 + * @param value 实际值 + * @return 类型错误 + */ + private static IllegalStateException invalidFlattenValue( + LoopResultReference reference, + int iterationIndex, + Object value) { + String actualType = value == null + ? "null" + : value.getClass().getName(); + String iteration = iterationIndex < 0 + ? "" + : ", iteration " + iterationIndex; + return new IllegalStateException( + "Loop output '" + reference.getOutputName() + + "' requires an array for flatten aggregation" + + iteration + + ", but got " + actualType); + } + /** * 循环结果批量读取分组键。 */ diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java index 7d5895f..0238105 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/LoopNode.java @@ -401,7 +401,10 @@ public class LoopNode extends BaseNode { prevTrigger.getLoopCursors().remove(this.id); } Map completedResult = chain.getLoopResultRepository().references( - loopContext.resultId, loopContext.currentIndex, getOutputNames()); + loopContext.resultId, + loopContext.currentIndex, + getOutputNames(), + getFlattenedOutputNames()); chain.getLoopResultRepository().releaseActiveCache( loopContext.resultId); if (!loopContext.inputExternalized) { @@ -1041,6 +1044,24 @@ public class LoopNode extends BaseNode { return outputNames; } + /** + * 获取启用一层扁平聚合的输出名称。 + * + * @return 保持定义顺序的输出名称集合 + */ + private Set getFlattenedOutputNames() { + Set outputNames = new LinkedHashSet<>(); + List outputDefs = getOutputDefs(); + if (outputDefs != null) { + for (Parameter outputDef : outputDefs) { + if (outputDef.isFlattenAggregation()) { + outputNames.add(outputDef.getName()); + } + } + } + return outputNames; + } + /** * 估算本轮输出占用字节数,用于宽松的累计结果失控保护。 * diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java index 046bb32..cbe5eab 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/BaseNodeParser.java @@ -76,6 +76,8 @@ public abstract class BaseNodeParser implements NodeParser", + "ref", + "knowledge.documents.content", + null); + output.put("flattenAggregation", true); + outputDefs.add(output); + data.put("outputDefs", outputDefs); + JSONObject nodeJson = new JSONObject(); + nodeJson.put("id", "loop"); + nodeJson.put("type", "loopNode"); + nodeJson.put("data", data); + + LoopNode loopNode = new LoopNodeParser().parse( + nodeJson, new JSONObject(), null); + + Assert.assertEquals(1, loopNode.getOutputDefs().size()); + Assert.assertTrue( + loopNode.getOutputDefs().get(0).isFlattenAggregation()); + } + /** * 构造参数 JSON。 * diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java index b60318a..ebf49fd 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/LoopResultReferenceResolverTest.java @@ -12,6 +12,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Iterator; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; /** @@ -42,6 +43,98 @@ public class LoopResultReferenceResolverTest { Assert.assertEquals(1, repository.getLoadCount()); } + /** + * 验证数组输出按轮次和轮内顺序只扁平一层。 + */ + @Test + public void shouldFlattenArrayOutputOnceInStableOrder() { + InMemoryLoopResultRepository repository = + new InMemoryLoopResultRepository(); + repository.append( + "result-flatten", + 0, + Map.of( + "items", + List.of( + List.of("a"), + List.of("b")))); + repository.append( + "result-flatten", + 1, + Map.of( + "items", + List.of( + List.of("c")))); + + Object resolved = repository.resolve( + new LoopResultReference( + "result-flatten", + 2, + "items", + true)); + + Assert.assertEquals( + List.of( + List.of("a"), + List.of("b"), + List.of("c")), + resolved); + } + + /** + * 验证启用扁平聚合后拒绝某轮标量结果。 + */ + @Test + public void shouldRejectScalarIterationWhenFlattening() { + InMemoryLoopResultRepository repository = + new InMemoryLoopResultRepository(); + repository.append( + "result-invalid", + 0, + Map.of("items", List.of("a"))); + repository.append( + "result-invalid", + 1, + Map.of("items", "scalar")); + + try { + repository.resolve( + new LoopResultReference( + "result-invalid", + 2, + "items", + true)); + Assert.fail("scalar iteration should be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue( + expected.getMessage().contains("items")); + Assert.assertTrue( + expected.getMessage().contains("iteration 1")); + } + } + + /** + * 验证循环完成时只给指定输出携带扁平聚合策略。 + */ + @Test + public void shouldCreateReferencesWithPerOutputAggregation() { + InMemoryLoopResultRepository repository = + new InMemoryLoopResultRepository(); + + Map references = repository.references( + "result-references", + 2, + List.of("grouped", "flattened"), + Set.of("flattened")); + + Assert.assertFalse( + ((LoopResultReference) references.get("grouped")) + .isFlattenAggregation()); + Assert.assertTrue( + ((LoopResultReference) references.get("flattened")) + .isFlattenAggregation()); + } + /** * 验证无限 Iterable 在达到预算后立即停止物化。 */ From fd3d9ad4196476cba210e15e2378748e645d10d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 31 Jul 2026 14:52:45 +0800 Subject: [PATCH 18/33] =?UTF-8?q?fix:=20=E8=84=B1=E6=95=8F=E5=A4=9A?= =?UTF-8?q?=E6=A8=A1=E6=80=81=E5=9B=BE=E7=89=87=E8=AF=B7=E6=B1=82=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 隐去图片 Data URI 编码正文并保留 MIME 与长度 - 补充日志脱敏和 image_url 序列化测试 --- .../core/model/chat/log/ChatLogSanitizer.java | 72 +++++++++++++++++++ .../chat/log/DefaultChatMessageLogger.java | 3 +- .../log/DefaultChatMessageLoggerTest.java | 32 +++++++++ .../OpenAIChatMessageSerializerTest.java | 36 ++++++++++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/ChatLogSanitizer.java create mode 100644 easy-agents-core/src/test/java/com/easyagents/core/model/chat/log/DefaultChatMessageLoggerTest.java create mode 100644 easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/ChatLogSanitizer.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/ChatLogSanitizer.java new file mode 100644 index 0000000..700ee4e --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/ChatLogSanitizer.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + */ +package com.easyagents.core.model.chat.log; + +/** + * 清理聊天请求日志中的大体积或敏感媒体内容。 + */ +final class ChatLogSanitizer { + + private static final String DATA_URI_PREFIX = "data:image/"; + private static final String BASE64_MARKER = ";base64,"; + + private ChatLogSanitizer() { + } + + /** + * 隐去图片 Data URI 的编码正文,仅保留 MIME 和字符长度。 + * + * @param message 原始日志消息 + * @return 脱敏后的日志消息 + */ + static String redactImageDataUris(String message) { + if (message == null || message.indexOf(DATA_URI_PREFIX) < 0) { + return message; + } + + StringBuilder sanitized = new StringBuilder(message.length()); + int cursor = 0; + while (cursor < message.length()) { + int dataUriStart = message.indexOf(DATA_URI_PREFIX, cursor); + if (dataUriStart < 0) { + sanitized.append(message, cursor, message.length()); + break; + } + int base64Marker = message.indexOf(BASE64_MARKER, dataUriStart); + if (base64Marker < 0) { + sanitized.append(message, cursor, message.length()); + break; + } + int payloadStart = base64Marker + BASE64_MARKER.length(); + int payloadEnd = payloadStart; + while (payloadEnd < message.length() && isBase64Character(message.charAt(payloadEnd))) { + payloadEnd++; + } + + sanitized.append(message, cursor, payloadStart); + sanitized.append("<已脱敏,编码长度=").append(payloadEnd - payloadStart).append('>'); + cursor = payloadEnd; + } + return sanitized.toString(); + } + + /** + * 判断字符是否可能属于 Base64 正文。 + * + * @param value 待判断字符 + * @return 是否属于 Base64 字符集 + */ + private static boolean isBase64Character(char value) { + return value >= 'A' && value <= 'Z' + || value >= 'a' && value <= 'z' + || value >= '0' && value <= '9' + || value == '+' + || value == '/' + || value == '=' + || value == '-' + || value == '_'; + } +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/DefaultChatMessageLogger.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/DefaultChatMessageLogger.java index 809a75e..0f328e9 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/DefaultChatMessageLogger.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/log/DefaultChatMessageLogger.java @@ -36,7 +36,8 @@ public class DefaultChatMessageLogger implements IChatMessageLogger { if (shouldLog(config)) { String provider = getProviderName(config); String model = getModelName(config); - logConsumer.accept(String.format("[%s/%s] >>>> request: %s", provider, model, message)); + logConsumer.accept(String.format("[%s/%s] >>>> request: %s", + provider, model, ChatLogSanitizer.redactImageDataUris(message))); } } diff --git a/easy-agents-core/src/test/java/com/easyagents/core/model/chat/log/DefaultChatMessageLoggerTest.java b/easy-agents-core/src/test/java/com/easyagents/core/model/chat/log/DefaultChatMessageLoggerTest.java new file mode 100644 index 0000000..7e7fd95 --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/model/chat/log/DefaultChatMessageLoggerTest.java @@ -0,0 +1,32 @@ +package com.easyagents.core.model.chat.log; + +import com.easyagents.core.model.chat.ChatConfig; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * 默认聊天日志记录器测试。 + */ +public class DefaultChatMessageLoggerTest { + + /** + * 验证请求日志不会输出完整图片 Base64。 + */ + @Test + public void shouldRedactImageDataUriPayload() { + AtomicReference logged = new AtomicReference<>(); + DefaultChatMessageLogger logger = new DefaultChatMessageLogger(logged::set); + ChatConfig config = new ChatConfig(); + config.setProvider("test"); + config.setModel("vision"); + + logger.logRequest(config, + "{\"url\":\"data:image/png;base64,AQIDBA==\",\"text\":\"ok\"}"); + + Assert.assertNotNull(logged.get()); + Assert.assertTrue(logged.get().contains("data:image/png;base64,<已脱敏,编码长度=8>")); + Assert.assertFalse(logged.get().contains("AQIDBA==")); + } +} diff --git a/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java b/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java new file mode 100644 index 0000000..6fa0b24 --- /dev/null +++ b/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java @@ -0,0 +1,36 @@ +package com.easyagents.core.test.model.client; + +import com.easyagents.core.message.UserMessage; +import com.easyagents.core.model.chat.ChatConfig; +import com.easyagents.core.model.client.OpenAIChatMessageSerializer; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +/** + * OpenAI-compatible 多模态消息序列化测试。 + */ +public class OpenAIChatMessageSerializerTest { + + /** + * 验证 Data URI 会写入标准的 image_url.url 字段。 + */ + @Test + public void shouldSerializeImageDataUriIntoImageUrlField() { + String dataUri = "data:image/png;base64,AQID"; + UserMessage message = new UserMessage("识别图片"); + message.addImageUrl(dataUri); + + List> messages = new OpenAIChatMessageSerializer() + .serializeMessages(List.of(message), new ChatConfig()); + + Assert.assertNotNull(messages); + List content = (List) messages.get(0).get("content"); + Map imageContent = (Map) content.get(1); + Map imageUrl = (Map) imageContent.get("image_url"); + Assert.assertEquals("image_url", imageContent.get("type")); + Assert.assertEquals(dataUri, imageUrl.get("url")); + } +} From f0a5aacc9281c0416f31c8334def04f2286aba6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 3 Aug 2026 11:17:44 +0800 Subject: [PATCH 19/33] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20JavaScript?= =?UTF-8?q?=20=E4=BB=A3=E7=A0=81=E8=8A=82=E7=82=B9=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E4=B8=8E=E9=94=99=E8=AF=AF=E5=AE=9A=E4=BD=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 复用 Graal Engine 与分段有界 Source 缓存,保持执行 Context 隔离 - 增加并发超时取消、脚本行列错误及参数解析复用 --- .../flow/core/code/CodeRuntimeEngine.java | 31 ++ .../core/code/CodeRuntimeEngineManager.java | 41 ++- .../impl/JavascriptExecutionException.java | 34 ++ .../code/impl/JavascriptRuntimeEngine.java | 318 +++++++++++++++++- .../easyagents/flow/core/node/CodeNode.java | 6 +- .../test/JavascriptRuntimeEngineTest.java | 257 ++++++++++++++ 6 files changed, 669 insertions(+), 18 deletions(-) create mode 100644 easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptExecutionException.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngine.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngine.java index 21a853e..e949fda 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngine.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngine.java @@ -20,6 +20,37 @@ import com.easyagents.flow.core.node.CodeNode; import java.util.Map; +/** + * 代码节点运行时引擎。 + */ public interface CodeRuntimeEngine { + + /** + * 执行代码节点。 + * + * @param code 解析模板后的代码 + * @param node 当前代码节点 + * @param chain 当前工作流 + * @return 节点输出 + */ Map execute(String code, CodeNode node, Chain chain); + + /** + * 使用已经解析的节点参数执行代码。 + * + *

默认委托旧接口,保持自定义运行时引擎兼容。内置引擎可覆盖该方法, + * 避免重复解析节点参数。

+ * + * @param code 解析模板后的代码 + * @param node 当前代码节点 + * @param chain 当前工作流 + * @param parameterValues 已解析的节点参数 + * @return 节点输出 + */ + default Map execute(String code, + CodeNode node, + Chain chain, + Map parameterValues) { + return execute(code, node, chain); + } } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngineManager.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngineManager.java index 23d6b9a..0769d3f 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngineManager.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/CodeRuntimeEngineManager.java @@ -20,16 +20,23 @@ import com.easyagents.flow.core.code.impl.JavascriptRuntimeEngine; import java.util.ArrayList; import java.util.List; +/** + * 管理代码节点运行时引擎提供者。 + */ public class CodeRuntimeEngineManager { public List providers = new ArrayList<>(); + /** + * JavaScript 引擎实例,可在业务启动阶段配置超时后替换。 + */ + private volatile JavascriptRuntimeEngine javascriptRuntimeEngine; private static class ManagerHolder { private static final CodeRuntimeEngineManager INSTANCE = new CodeRuntimeEngineManager(); } private CodeRuntimeEngineManager() { - JavascriptRuntimeEngine javascriptRuntimeEngine = new JavascriptRuntimeEngine(); + javascriptRuntimeEngine = new JavascriptRuntimeEngine(); providers.add(engineId -> { if ("js".equals(engineId) || "javascript".equals(engineId)) { return javascriptRuntimeEngine; @@ -38,18 +45,50 @@ public class CodeRuntimeEngineManager { }); } + /** + * 获取全局代码运行时管理器。 + * + * @return 代码运行时管理器 + */ public static CodeRuntimeEngineManager getInstance() { return ManagerHolder.INSTANCE; } + /** + * 配置 JavaScript 代码执行超时。 + * + * @param timeoutMs 超时时间;0 表示关闭超时 + * @throws IllegalArgumentException 超时时间为负数时抛出 + */ + public void configureJavascriptRuntimeEngine(long timeoutMs) { + javascriptRuntimeEngine = + new JavascriptRuntimeEngine(timeoutMs); + } + + /** + * 注册代码运行时提供者。 + * + * @param provider 代码运行时提供者 + */ public void registerProvider(CodeRuntimeEngineProvider provider) { providers.add(provider); } + /** + * 移除代码运行时提供者。 + * + * @param provider 代码运行时提供者 + */ public void removeProvider(CodeRuntimeEngineProvider provider) { providers.remove(provider); } + /** + * 按引擎标识获取代码运行时。 + * + * @param engineId 引擎标识 + * @return 匹配的代码运行时;未找到时返回 {@code null} + */ public CodeRuntimeEngine getCodeRuntimeEngine(Object engineId) { for (CodeRuntimeEngineProvider provider : providers) { CodeRuntimeEngine codeRuntimeEngine = provider.getCodeRuntimeEngine(engineId); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptExecutionException.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptExecutionException.java new file mode 100644 index 0000000..3758c5a --- /dev/null +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptExecutionException.java @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2025-2026, Michael Yang 杨福海 (fuhai999@gmail.com). + *

+ * Licensed under the GNU Lesser General Public License (LGPL) ,Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.gnu.org/licenses/lgpl-3.0.txt + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.flow.core.code.impl; + +/** + * 可直接反馈给工作流使用者的 JavaScript 执行异常。 + */ +public class JavascriptExecutionException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + /** + * 创建 JavaScript 执行异常。 + * + * @param message 面向使用者的错误信息 + * @param cause 原始执行异常 + */ + public JavascriptExecutionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java index 6496981..76a8a69 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java @@ -23,11 +23,23 @@ import com.easyagents.flow.core.code.CodeRuntimeEngine; import com.easyagents.flow.core.node.CodeNode; import com.easyagents.flow.core.util.graalvm.JsInteropUtils; import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Engine; import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.SourceSection; import org.graalvm.polyglot.Value; +import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; /** * 基于 GraalVM 的 JavaScript 代码节点执行器。 @@ -38,26 +50,93 @@ import java.util.Map; */ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { - // 使用 Context.Builder 构建上下文,线程安全 - private static final Context.Builder CONTEXT_BUILDER = Context.newBuilder("js") + private static final int SOURCE_CACHE_LIMIT = 512; + private static final int SOURCE_CACHE_SEGMENT_COUNT = 16; + private static final int SOURCE_CACHE_SEGMENT_LIMIT = + SOURCE_CACHE_LIMIT / SOURCE_CACHE_SEGMENT_COUNT; + private static final int MAX_CACHEABLE_SOURCE_CHARS = 16 * 1024; + private static final int TIMEOUT_EXECUTOR_THREADS = + Math.max(2, Math.min( + Runtime.getRuntime().availableProcessors(), 8)); + private static final String SOURCE_NAME = + "workflow-code-node.js"; + private static final AtomicInteger TIMEOUT_THREAD_SEQUENCE = + new AtomicInteger(); + /** + * Engine 跨隔离 Context 共享编译缓存。 + */ + private static final Engine ENGINE = Engine.newBuilder() .option("engine.WarnInterpreterOnly", "false") - .allowHostAccess(HostAccess.ALL) // 允许访问 Java 对象的方法和字段 - .allowHostClassLookup(className -> false) // 禁止动态加载任意 Java 类 - .option("js.ecmascript-version", "2021"); // 使用较新的 ECMAScript 版本 - + .build(); + private static final Source RESULT_INIT_SOURCE = + Source.create("js", "var _result = {};"); + private static final List> SOURCE_CACHE_SEGMENTS = + createSourceCacheSegments(); + private static final ScheduledThreadPoolExecutor + TIMEOUT_EXECUTOR = createTimeoutExecutor(); /** - * 执行 JavaScript 代码并返回节点输出。 + * 单次 JavaScript 执行超时,0 表示关闭。 + */ + private final long timeoutMs; + + /** + * 创建不启用超时的 JavaScript 执行器。 + */ + public JavascriptRuntimeEngine() { + this(0L); + } + + /** + * 创建 JavaScript 执行器。 + * + * @param timeoutMs 单次执行超时,0 表示关闭 + * @throws IllegalArgumentException 超时时间为负数时抛出 + */ + public JavascriptRuntimeEngine(long timeoutMs) { + if (timeoutMs < 0L) { + throw new IllegalArgumentException( + "JavaScript 执行超时时间不能为负数"); + } + this.timeoutMs = timeoutMs; + } + + /** + * 执行 JavaScript 代码并返回节点输出,兼容直接调用旧接口的场景。 * * @param code 用户代码 * @param node 当前代码节点 * @param chain 当前工作流 * @return 代码节点输出 - * @throws RuntimeException JavaScript 执行失败或 main 返回值不是对象时抛出 + * @throws JavascriptExecutionException JavaScript 执行失败时抛出 */ @Override public Map execute(String code, CodeNode node, Chain chain) { - try (Context context = CONTEXT_BUILDER.build()) { + Map parameterValues = + chain.getExecutionState().resolveParameters(node); + return execute(code, node, chain, parameterValues); + } + + /** + * 使用已经解析的节点参数执行 JavaScript 代码。 + * + * @param code 用户代码 + * @param node 当前代码节点 + * @param chain 当前工作流 + * @param parameterValues 已解析的节点参数 + * @return 代码节点输出 + * @throws JavascriptExecutionException JavaScript 执行失败时抛出 + */ + @Override + public Map execute(String code, + CodeNode node, + Chain chain, + Map parameterValues) { + Context context = createContext(); + AtomicBoolean timedOut = new AtomicBoolean(false); + ScheduledFuture timeoutFuture = + scheduleTimeout(context, timedOut); + try (Context ignored = context) { Value bindings = context.getBindings("js"); ChainState chainState = chain.getExecutionState(); @@ -72,9 +151,6 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { } }); - // 注入参数 - Map parameterValues = - chainState.resolveParameters(node); if (parameterValues != null) { for (Map.Entry entry : parameterValues.entrySet()) { bindings.putMember(entry.getKey(), JsInteropUtils.wrapJavaValueForJS(context, entry.getValue())); @@ -82,22 +158,232 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { } // 在 JS 中创建 _result 对象 - context.eval("js", "var _result = {};"); + context.eval(RESULT_INIT_SOURCE); // 注入 _chain 和 _context bindings.putMember("_chain", chain); bindings.putMember("_state", nodeState); // 执行用户脚本 - context.eval("js", code); + context.eval(source(code, node)); return resolveResult(context, bindings, parameterValues); - + } catch (PolyglotException e) { + throw executionException(e, timedOut.get()); } catch (Exception e) { - throw new RuntimeException("Polyglot JS 脚本执行失败: " + e.getMessage(), e); + if (timedOut.get()) { + throw timeoutException(e); + } + throw new JavascriptExecutionException( + "JavaScript 执行失败:" + safeMessage(e), + e); + } finally { + if (timeoutFuture != null) { + timeoutFuture.cancel(false); + } } } + /** + * 创建一次隔离的 JavaScript Context。 + * + * @return 新的 JavaScript Context + */ + private Context createContext() { + return Context.newBuilder("js") + .engine(ENGINE) + .allowHostAccess(HostAccess.ALL) + .allowHostClassLookup(className -> false) + .option("js.ecmascript-version", "2021") + .build(); + } + + /** + * 获取可复用的用户脚本 Source。 + * + *

模板化脚本可能随输入生成大量不同源码,因此不进入共享缓存。 + * 超长脚本同样跳过缓存,限制源码与编译元数据占用。

+ * + * @param code 解析模板后的代码 + * @param node 当前代码节点 + * @return GraalVM Source + */ + private Source source(String code, CodeNode node) { + String originalCode = + node == null ? null : node.getCode(); + boolean dynamicTemplate = originalCode != null + && originalCode.contains("{{"); + if (dynamicTemplate + || code.length() > MAX_CACHEABLE_SOURCE_CHARS) { + return Source.newBuilder("js", code, SOURCE_NAME) + .cached(false) + .buildLiteral(); + } + Map sourceCache = + SOURCE_CACHE_SEGMENTS.get( + Math.floorMod( + code.hashCode(), + SOURCE_CACHE_SEGMENT_COUNT)); + synchronized (sourceCache) { + return sourceCache.computeIfAbsent( + code, + script -> Source.newBuilder( + "js", + script, + SOURCE_NAME) + .cached(true) + .buildLiteral()); + } + } + + /** + * 创建分段有界的 Source 缓存。 + * + *

每个分段独立维护 LRU 淘汰,避免所有工作流竞争同一把锁, + * 同时保证总缓存条目不超过全局上限。

+ * + * @return Source 缓存分段 + */ + private static List> + createSourceCacheSegments() { + List> segments = + new ArrayList<>(SOURCE_CACHE_SEGMENT_COUNT); + for (int index = 0; + index < SOURCE_CACHE_SEGMENT_COUNT; + index++) { + segments.add(new LinkedHashMap<>( + SOURCE_CACHE_SEGMENT_LIMIT + 1, + 0.75F, + true) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + return size() + > SOURCE_CACHE_SEGMENT_LIMIT; + } + }); + } + return Collections.unmodifiableList(segments); + } + + /** + * 安排单次执行超时。 + * + * @param context 待取消的 Context + * @param timedOut 超时状态 + * @return 超时任务;关闭超时时返回 {@code null} + */ + private ScheduledFuture scheduleTimeout( + Context context, + AtomicBoolean timedOut) { + if (timeoutMs <= 0L) { + return null; + } + return TIMEOUT_EXECUTOR.schedule(() -> { + timedOut.set(true); + context.close(true); + }, timeoutMs, TimeUnit.MILLISECONDS); + } + + /** + * 将 GraalVM 异常转换为用户可定位的脚本异常。 + * + * @param error GraalVM 异常 + * @param timedOut 是否由超时任务取消 + * @return JavaScript 执行异常 + */ + private JavascriptExecutionException executionException( + PolyglotException error, + boolean timedOut) { + if (timedOut) { + return timeoutException(error); + } + String errorType = error.isSyntaxError() + ? "JavaScript 语法错误" + : "JavaScript 执行失败"; + SourceSection location = sourceLocation(error); + String locationText = location == null + ? "" + : "(第 " + location.getStartLine() + + " 行,第 " + location.getStartColumn() + " 列)"; + return new JavascriptExecutionException( + errorType + locationText + ":" + safeMessage(error), + error); + } + + /** + * 获取异常对应的用户脚本位置。 + * + *

部分 GraalVM 版本不会在运行时异常上直接设置位置, + * 此时从首个 guest stack frame 补取。

+ * + * @param error GraalVM 异常 + * @return 用户脚本位置;无法定位时返回 {@code null} + */ + private SourceSection sourceLocation(PolyglotException error) { + SourceSection location = error.getSourceLocation(); + if (location != null) { + return location; + } + for (PolyglotException.StackFrame frame + : error.getPolyglotStackTrace()) { + if (frame.isGuestFrame() + && frame.getSourceLocation() != null) { + return frame.getSourceLocation(); + } + } + return null; + } + + /** + * 创建 JavaScript 执行超时异常。 + * + * @param cause 超时取消产生的异常 + * @return JavaScript 执行异常 + */ + private JavascriptExecutionException timeoutException( + Throwable cause) { + return new JavascriptExecutionException( + "JavaScript 执行超时(" + timeoutMs + "ms)", + cause); + } + + /** + * 获取稳定的异常消息。 + * + * @param error 原始异常 + * @return 非空异常消息 + */ + private String safeMessage(Throwable error) { + String message = error.getMessage(); + return message == null || message.isBlank() + ? error.getClass().getSimpleName() + : message; + } + + /** + * 创建超时调度器。 + * + * @return 有界多线程守护调度器 + */ + private static ScheduledThreadPoolExecutor + createTimeoutExecutor() { + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor( + TIMEOUT_EXECUTOR_THREADS, + task -> { + Thread thread = new Thread( + task, + "javascript-runtime-timeout-" + + TIMEOUT_THREAD_SEQUENCE + .incrementAndGet()); + thread.setDaemon(true); + return thread; + }); + executor.setRemoveOnCancelPolicy(true); + return executor; + } + /** * 解析脚本输出,兼容历史 _result 写法并自动调用 main 函数。 * diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java index 09e1f3f..e49d839 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java @@ -63,7 +63,11 @@ public class CodeNode extends BaseNode { if (codeRuntimeEngine == null) { throw new IllegalArgumentException("code runtime engine not found: " + this.engine); } - return codeRuntimeEngine.execute(newCode, this, chain); + return codeRuntimeEngine.execute( + newCode, + this, + chain, + parameterValues); } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java index cf675e4..982b534 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java @@ -6,14 +6,25 @@ import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.code.impl.JavascriptExecutionException; +import com.easyagents.flow.core.code.impl.JavascriptRuntimeEngine; import com.easyagents.flow.core.node.CodeNode; import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -82,6 +93,252 @@ public class JavascriptRuntimeEngineTest { } } + /** + * 验证语法错误包含可定位的脚本行列信息。 + */ + @Test + public void shouldExposeSyntaxErrorLocation() { + CodeNode node = codeNode("const answer = ;"); + Chain chain = chain(Collections.emptyMap()); + + try { + node.execute(chain); + Assert.fail("JavaScript 语法错误应执行失败"); + } catch (JavascriptExecutionException exception) { + Assert.assertTrue(exception.getMessage().contains( + "JavaScript 语法错误")); + Assert.assertTrue(exception.getMessage().contains("第 1 行")); + } + } + + /** + * 验证运行时异常包含原始错误和脚本位置。 + */ + @Test + public void shouldExposeRuntimeErrorLocation() { + CodeNode node = codeNode("throw new Error('boom');"); + Chain chain = chain(Collections.emptyMap()); + + try { + node.execute(chain); + Assert.fail("JavaScript 运行时异常应执行失败"); + } catch (JavascriptExecutionException exception) { + Assert.assertTrue(exception.getMessage().contains( + "JavaScript 执行失败")); + Assert.assertTrue(exception.getMessage().contains("第 1 行")); + Assert.assertTrue(exception.getMessage().contains("boom")); + } + } + + /** + * 验证无限循环会在配置时间内被取消。 + */ + @Test + public void shouldCancelExecutionAfterTimeout() { + JavascriptRuntimeEngine engine = + new JavascriptRuntimeEngine(100L); + CodeNode node = codeNode("while (true) {}"); + Chain chain = chain(Collections.emptyMap()); + long startedAt = System.currentTimeMillis(); + + try { + engine.execute( + node.getCode(), + node, + chain, + Collections.emptyMap()); + Assert.fail("JavaScript 超时应执行失败"); + } catch (JavascriptExecutionException exception) { + Assert.assertTrue(exception.getMessage().contains( + "JavaScript 执行超时(100ms)")); + Assert.assertTrue( + "超时取消应在合理时间内完成", + System.currentTimeMillis() - startedAt < 3000L); + } + } + + /** + * 验证负数超时不会静默关闭执行保护。 + */ + @Test + public void shouldRejectNegativeTimeout() { + try { + new JavascriptRuntimeEngine(-1L); + Assert.fail("负数超时应被拒绝"); + } catch (IllegalArgumentException exception) { + Assert.assertTrue(exception.getMessage().contains( + "不能为负数")); + } + } + + /** + * 验证多个无限循环可并发触发超时取消。 + * + * @throws Exception 并发任务执行失败时抛出 + */ + @Test + public void shouldCancelConcurrentExecutionsAfterTimeout() + throws Exception { + int executionCount = 8; + JavascriptRuntimeEngine engine = + new JavascriptRuntimeEngine(100L); + ExecutorService executor = + Executors.newFixedThreadPool(executionCount); + CountDownLatch ready = + new CountDownLatch(executionCount); + CountDownLatch start = new CountDownLatch(1); + List> futures = + new ArrayList<>(executionCount); + try { + for (int index = 0; + index < executionCount; + index++) { + final int executionIndex = index; + futures.add(executor.submit(() -> { + CodeNode node = codeNode("while (true) {}"); + node.setId("timeout-node-" + executionIndex); + Chain chain = chain(Collections.emptyMap()); + ready.countDown(); + start.await(); + try { + engine.execute( + node.getCode(), + node, + chain, + Collections.emptyMap()); + return "未触发超时"; + } catch (JavascriptExecutionException exception) { + return exception.getMessage(); + } + })); + } + Assert.assertTrue( + "并发执行未及时就绪", + ready.await(2L, TimeUnit.SECONDS)); + long startedAt = System.currentTimeMillis(); + start.countDown(); + for (Future future : futures) { + Assert.assertTrue( + future.get(3L, TimeUnit.SECONDS) + .contains("JavaScript 执行超时(100ms)")); + } + Assert.assertTrue( + "并发超时取消应在合理时间内完成", + System.currentTimeMillis() - startedAt < 3000L); + } finally { + start.countDown(); + executor.shutdownNow(); + } + } + + /** + * 验证分段 Source 缓存在并发未命中时保持结果正确。 + * + * @throws Exception 并发任务执行失败时抛出 + */ + @Test + public void shouldExecuteCacheMissesConcurrently() + throws Exception { + int executionCount = 32; + JavascriptRuntimeEngine engine = + new JavascriptRuntimeEngine(); + ExecutorService executor = + Executors.newFixedThreadPool(8); + List> futures = + new ArrayList<>(executionCount); + try { + for (int index = 0; + index < executionCount; + index++) { + final int expected = index; + futures.add(executor.submit(() -> { + CodeNode node = codeNode( + "_result.value = " + expected + ";"); + node.setId("cache-node-" + expected); + Map result = engine.execute( + node.getCode(), + node, + chain(Collections.emptyMap()), + Collections.emptyMap()); + return ((Number) result.get("value")) + .longValue(); + })); + } + for (int index = 0; + index < executionCount; + index++) { + Assert.assertEquals( + index, + futures.get(index) + .get(3L, TimeUnit.SECONDS) + .longValue()); + } + } finally { + executor.shutdownNow(); + } + } + + /** + * 验证分段缓存总条目数始终受上限约束。 + * + * @throws Exception 反射访问缓存失败时抛出 + */ + @Test + @SuppressWarnings("unchecked") + public void shouldKeepSourceCacheBounded() + throws Exception { + JavascriptRuntimeEngine engine = + new JavascriptRuntimeEngine(); + Method sourceMethod = JavascriptRuntimeEngine.class + .getDeclaredMethod( + "source", + String.class, + CodeNode.class); + sourceMethod.setAccessible(true); + CodeNode node = codeNode(""); + for (int index = 0; index < 1024; index++) { + String code = "_result.value = " + index + ";"; + node.setCode(code); + sourceMethod.invoke(engine, code, node); + } + + Field cacheField = JavascriptRuntimeEngine.class + .getDeclaredField("SOURCE_CACHE_SEGMENTS"); + cacheField.setAccessible(true); + List> segments = + (List>) cacheField.get(null); + int cachedSourceCount = segments.stream() + .mapToInt(Map::size) + .sum(); + + Assert.assertTrue(cachedSourceCount <= 512); + Assert.assertTrue( + segments.stream() + .allMatch(segment -> + segment.size() <= 32)); + } + + /** + * 验证共享 Engine 下不同执行仍保持全局变量隔离。 + */ + @Test + public void shouldKeepExecutionContextsIsolated() { + CodeNode node = codeNode(String.join("\n", + "if (typeof globalCounter === 'undefined') {", + " globalCounter = 0;", + "}", + "globalCounter += 1;", + "_result.counter = globalCounter;")); + + Map first = + node.execute(chain(Collections.emptyMap())); + Map second = + node.execute(chain(Collections.emptyMap())); + + Assert.assertEquals(1L, first.get("counter")); + Assert.assertEquals(1L, second.get("counter")); + } + /** * 创建 JavaScript 代码节点。 * From 15adcfff428a61bbd56d97cff0b219aeb356f688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 3 Aug 2026 11:32:25 +0800 Subject: [PATCH 20/33] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20MinerU=20?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E4=BB=BB=E5=8A=A1=E6=8F=90=E4=BA=A4=E6=8E=A2?= =?UTF-8?q?=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用健康接口预检服务可用性并缓存短期故障 - 限制任务提交总时长并快速处理服务端错误 - 补齐 Starter 配置传递与核心回归测试 --- .../document/core/mineru/MineruClient.java | 134 +++++++++++- .../core/mineru/MineruProperties.java | 19 ++ .../MineruDocumentParseServiceTest.java | 201 ++++++++++++++++++ .../CommonMineruDocumentProperties.java | 19 ++ .../mineru/MineruPdfAutoConfiguration.java | 1 + .../pptx/MineruPptxAutoConfiguration.java | 1 + .../xlsx/MineruXlsxAutoConfiguration.java | 1 + 7 files changed, 373 insertions(+), 3 deletions(-) diff --git a/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruClient.java b/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruClient.java index be2fb6a..216921c 100644 --- a/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruClient.java +++ b/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruClient.java @@ -5,6 +5,7 @@ import com.easyagents.core.util.StringUtil; import com.easyagents.document.core.exception.DocumentParseException; import com.easyagents.document.core.entity.ParseFile; import com.easyagents.document.core.entity.ParseRequest; +import okhttp3.Call; import okhttp3.MediaType; import okhttp3.MultipartBody; import okhttp3.OkHttpClient; @@ -29,10 +30,17 @@ import java.util.concurrent.TimeUnit; public class MineruClient { private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.parse("application/octet-stream"); + private static final int DEFAULT_SUBMIT_TIMEOUT_MS = 120000; + private static final int AVAILABILITY_PROBE_TIMEOUT_MS = 3000; + private static final long AVAILABILITY_CACHE_TTL_MS = 5000L; private final String baseUrl; private final OkHttpClient okHttpClient; private final MineruMapper mineruMapper; + private final int submitTimeoutMs; + private final Object availabilityProbeMonitor = new Object(); + private volatile long availabilityCacheDeadlineMs; + private volatile String availabilityFailureMessage; /** * 创建客户端。 @@ -66,6 +74,7 @@ public class MineruClient { this.baseUrl = normalizeBaseUrl(properties.getBaseUrl()); this.okHttpClient = okHttpClient; this.mineruMapper = mineruMapper; + this.submitTimeoutMs = positiveOrDefault(properties.getSubmitTimeoutMs(), DEFAULT_SUBMIT_TIMEOUT_MS); } /** @@ -85,7 +94,10 @@ public class MineruClient { * @return 原始任务状态 */ public MineruTaskStatus submit(ParseRequest request) { - return mineruMapper.toTaskStatus(executeJsonMultipart("/tasks", request, buildAsyncFormFields(request))); + assertServiceAvailable(); + return mineruMapper.toTaskStatus( + executeJsonMultipart("/tasks", request, buildAsyncFormFields(request), submitTimeoutMs) + ); } /** @@ -131,6 +143,22 @@ public class MineruClient { } protected JSONObject executeJsonMultipart(String path, ParseRequest request, Map> fields) { + return executeJsonMultipart(path, request, fields, 0); + } + + /** + * 执行带整次调用超时的 Multipart JSON 请求。 + * + * @param path 接口路径 + * @param request 解析请求 + * @param fields 表单字段 + * @param callTimeoutMs 整次调用超时时间,单位毫秒;小于等于 0 时沿用客户端阶段超时 + * @return JSON 响应 + */ + protected JSONObject executeJsonMultipart(String path, + ParseRequest request, + Map> fields, + long callTimeoutMs) { MultipartBody.Builder formBuilder = new MultipartBody.Builder().setType(MultipartBody.FORM); appendFiles(formBuilder, request.getFiles()); appendStringFields(formBuilder, fields); @@ -138,7 +166,9 @@ public class MineruClient { .url(baseUrl + path) .post(formBuilder.build()) .build(); - return executeJsonRequest(path, httpRequest); + return callTimeoutMs > 0 + ? executeJsonRequest(path, httpRequest, callTimeoutMs) + : executeJsonRequest(path, httpRequest); } protected JSONObject executeJsonGet(String path) { @@ -147,7 +177,28 @@ public class MineruClient { } protected JSONObject executeJsonRequest(String path, Request request) { - try (Response response = okHttpClient.newCall(request).execute()) { + return executeJsonRequest(path, request, 0); + } + + /** + * 执行 JSON 请求,并可限制连接、写入和读取在内的整次调用时长。 + * + * @param path 接口路径 + * @param request HTTP 请求 + * @param callTimeoutMs 整次调用超时时间,单位毫秒;小于等于 0 时不额外限制 + * @return JSON 响应 + */ + protected JSONObject executeJsonRequest(String path, Request request, long callTimeoutMs) { + Call call = okHttpClient.newCall(request); + if (callTimeoutMs > 0) { + // OkHttp 的 Call timeout 到期后会取消底层请求,避免线程长期阻塞在文件上传或响应等待。 + call.timeout().timeout(callTimeoutMs, TimeUnit.MILLISECONDS); + } + try (Response response = call.execute()) { + if (response.code() >= 500) { + // 服务端错误在响应头到达时立即失败,避免异常响应体未结束时继续占用提交线程。 + throw buildHttpException(path, response.code(), new byte[0]); + } ResponseBody body = response.body(); String bodyText = body == null ? "" : body.string(); if (!response.isSuccessful()) { @@ -163,6 +214,83 @@ public class MineruClient { } } + /** + * 在上传文件前探测 MinerU 网关是否可用,避免服务已返回 5xx 时仍传输大文件。 + * + *

探测结果短暂缓存,限制批量导入期间的额外请求量。健康检查返回 4xx 说明网关仍可达, + * 实际任务接口会继续完成业务校验。

+ * + * @throws DocumentParseException 网关返回 5xx 或探测请求失败 + */ + private void assertServiceAvailable() { + long now = System.currentTimeMillis(); + if (now < availabilityCacheDeadlineMs) { + throwCachedAvailabilityFailure(); + return; + } + synchronized (availabilityProbeMonitor) { + now = System.currentTimeMillis(); + if (now < availabilityCacheDeadlineMs) { + throwCachedAvailabilityFailure(); + return; + } + String healthPath = "/health"; + Request request = new Request.Builder().url(baseUrl + healthPath).get().build(); + Call call = okHttpClient.newCall(request); + call.timeout().timeout( + Math.min(submitTimeoutMs, AVAILABILITY_PROBE_TIMEOUT_MS), + TimeUnit.MILLISECONDS + ); + try (Response response = call.execute()) { + if (response.code() >= 500) { + cacheAvailabilityFailure( + "MinerU service unavailable: path=" + healthPath + ", status=" + response.code(), + now + ); + throw new DocumentParseException(availabilityFailureMessage); + } + availabilityFailureMessage = null; + availabilityCacheDeadlineMs = now + AVAILABILITY_CACHE_TTL_MS; + } catch (IOException exception) { + cacheAvailabilityFailure("MinerU service unavailable: availability probe failed", now); + throw new DocumentParseException(availabilityFailureMessage, exception); + } + } + } + + /** + * 缓存 MinerU 不可用状态。 + * + * @param message 失败信息 + * @param detectedAtMs 检测时间,单位毫秒 + */ + private void cacheAvailabilityFailure(String message, long detectedAtMs) { + availabilityFailureMessage = message; + availabilityCacheDeadlineMs = detectedAtMs + AVAILABILITY_CACHE_TTL_MS; + } + + /** + * 若缓存记录为不可用则抛出稳定异常。 + * + * @throws DocumentParseException MinerU 仍处于不可用缓存窗口 + */ + private void throwCachedAvailabilityFailure() { + if (availabilityFailureMessage != null) { + throw new DocumentParseException(availabilityFailureMessage); + } + } + + /** + * 返回正整数配置,非法配置回退到缺省值。 + * + * @param value 配置值 + * @param defaultValue 缺省值 + * @return 可用的正整数 + */ + private int positiveOrDefault(Integer value, int defaultValue) { + return value == null || value <= 0 ? defaultValue : value; + } + private void appendFiles(MultipartBody.Builder formBuilder, List files) { if (files == null || files.isEmpty()) { throw new IllegalArgumentException("Parse request must contain at least one file"); diff --git a/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruProperties.java b/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruProperties.java index d45ab20..1c70fa8 100644 --- a/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruProperties.java +++ b/easy-agents-document/easy-agents-document-core/src/main/java/com/easyagents/document/core/mineru/MineruProperties.java @@ -16,6 +16,7 @@ public class MineruProperties { private Integer connectTimeoutMs = 3000; private Integer readTimeoutMs = 600000; private Integer writeTimeoutMs = 600000; + private Integer submitTimeoutMs = 120000; private Integer pollIntervalMs = 1000; private Integer resultTimeoutMs = 1800000; private String defaultBackend = "vlm-http-client"; @@ -56,6 +57,24 @@ public class MineruProperties { this.writeTimeoutMs = writeTimeoutMs; } + /** + * 获取异步任务提交总超时时间。 + * + * @return 提交总超时时间,单位毫秒 + */ + public Integer getSubmitTimeoutMs() { + return submitTimeoutMs; + } + + /** + * 设置异步任务提交总超时时间。 + * + * @param submitTimeoutMs 提交总超时时间,单位毫秒 + */ + public void setSubmitTimeoutMs(Integer submitTimeoutMs) { + this.submitTimeoutMs = submitTimeoutMs; + } + public Integer getPollIntervalMs() { return pollIntervalMs; } diff --git a/easy-agents-document/easy-agents-document-core/src/test/java/com/easyagents/document/core/mineru/MineruDocumentParseServiceTest.java b/easy-agents-document/easy-agents-document-core/src/test/java/com/easyagents/document/core/mineru/MineruDocumentParseServiceTest.java index cdcaeb8..916e47a 100644 --- a/easy-agents-document/easy-agents-document-core/src/test/java/com/easyagents/document/core/mineru/MineruDocumentParseServiceTest.java +++ b/easy-agents-document/easy-agents-document-core/src/test/java/com/easyagents/document/core/mineru/MineruDocumentParseServiceTest.java @@ -6,14 +6,30 @@ import com.easyagents.document.core.entity.ParseRequest; import com.easyagents.document.core.entity.ParseResponse; import com.easyagents.document.core.entity.ParseTaskInfo; import com.easyagents.document.core.entity.ParseTaskStatus; +import com.easyagents.document.core.exception.DocumentParseException; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; import okio.Buffer; +import okio.BufferedSource; +import okio.Okio; +import okio.Source; +import okio.Timeout; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -89,6 +105,191 @@ public class MineruDocumentParseServiceTest { Assert.assertTrue(client.lastMultipartBody.contains("\r\nen\r\n")); } + /** + * 验证异步任务提交总超时会取消无响应的 HTTP 调用。 + * + * @throws Exception 本地测试套接字异常 + */ + @Test + public void submitShouldCancelUnresponsiveCallAtConfiguredTimeout() throws Exception { + AtomicReference acceptedSocket = new AtomicReference(); + try (ServerSocket serverSocket = new ServerSocket(0)) { + Thread serverThread = new Thread(() -> { + try { + Socket socket = serverSocket.accept(); + acceptedSocket.set(socket); + while (socket.getInputStream().read() >= 0) { + // 持续读取请求但不返回响应,模拟 MinerU 提交接口失去响应。 + } + } catch (IOException ignore) { + // 客户端超时取消或测试关闭套接字后结束服务线程。 + } + }, "mineru-submit-timeout-test-server"); + serverThread.setDaemon(true); + serverThread.start(); + + MineruProperties properties = defaultProperties(); + properties.setBaseUrl("http://127.0.0.1:" + serverSocket.getLocalPort()); + properties.setSubmitTimeoutMs(200); + properties.setConnectTimeoutMs(5000); + properties.setReadTimeoutMs(5000); + properties.setWriteTimeoutMs(5000); + MineruClient client = new MineruClient(properties, new MineruMapper(properties)); + + long startedAt = System.currentTimeMillis(); + try { + client.submit(buildRequest()); + Assert.fail("Expected MinerU submit timeout"); + } catch (DocumentParseException expected) { + long elapsed = System.currentTimeMillis() - startedAt; + Assert.assertTrue("Submit call should be cancelled promptly, elapsed=" + elapsed, elapsed < 2000); + } finally { + Socket socket = acceptedSocket.get(); + if (socket != null) { + socket.close(); + } + serverThread.join(1000); + } + } + } + + /** + * 验证 MinerU 网关返回 5xx 时会在上传文件前立即失败,并复用短期不可用缓存。 + * + * @throws Exception 本地测试套接字异常 + */ + @Test + public void submitShouldFailBeforeMultipartUploadWhenProbeReturnsServerError() throws Exception { + AtomicInteger requestCount = new AtomicInteger(); + AtomicReference requestLine = new AtomicReference(); + Thread serverThread; + try (ServerSocket serverSocket = new ServerSocket(0)) { + serverThread = new Thread(() -> { + while (!serverSocket.isClosed()) { + try (Socket socket = serverSocket.accept(); + BufferedReader reader = new BufferedReader( + new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { + requestCount.incrementAndGet(); + requestLine.set(reader.readLine()); + String header; + while ((header = reader.readLine()) != null && !header.isEmpty()) { + // 读取完整请求头后再返回网关错误。 + } + byte[] body = "Service Unavailable".getBytes(StandardCharsets.UTF_8); + String responseHeaders = "HTTP/1.1 503 Service Unavailable\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Length: " + body.length + "\r\n" + + "Connection: close\r\n\r\n"; + socket.getOutputStream().write(responseHeaders.getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().write(body); + socket.getOutputStream().flush(); + } catch (IOException ignore) { + // 测试结束关闭 ServerSocket 后退出服务线程。 + } + } + }, "mineru-availability-probe-test-server"); + serverThread.setDaemon(true); + serverThread.start(); + + MineruProperties properties = defaultProperties(); + properties.setBaseUrl("http://127.0.0.1:" + serverSocket.getLocalPort()); + MineruClient client = new MineruClient(properties, new MineruMapper(properties)); + + long startedAt = System.currentTimeMillis(); + for (int attempt = 0; attempt < 2; attempt++) { + try { + client.submit(buildRequest()); + Assert.fail("Expected MinerU availability failure"); + } catch (DocumentParseException expected) { + Assert.assertTrue(expected.getMessage().contains("status=503")); + } + } + long elapsed = System.currentTimeMillis() - startedAt; + Assert.assertTrue("Server error should fail promptly, elapsed=" + elapsed, elapsed < 2000); + } + serverThread.join(1000); + + Assert.assertEquals("Cached failure should avoid repeated probes", 1, requestCount.get()); + Assert.assertTrue(requestLine.get().startsWith("GET /health HTTP/1.1")); + } + + /** + * 验证任务接口返回 5xx 后不会等待或读取异常响应体。 + */ + @Test + public void submitShouldThrowServerErrorWithoutReadingResponseBody() { + AtomicInteger requestCount = new AtomicInteger(); + AtomicReference errorBodyRead = new AtomicReference(false); + OkHttpClient httpClient = new OkHttpClient.Builder() + .addInterceptor(chain -> { + int currentRequest = requestCount.incrementAndGet(); + Response.Builder responseBuilder = new Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1); + if (currentRequest == 1) { + return responseBuilder + .code(200) + .message("OK") + .body(ResponseBody.create((MediaType) null, new byte[0])) + .build(); + } + ResponseBody trackingBody = new ResponseBody() { + + private final BufferedSource source = Okio.buffer(new Source() { + + @Override + public long read(Buffer sink, long byteCount) { + errorBodyRead.set(true); + return -1; + } + + @Override + public Timeout timeout() { + return Timeout.NONE; + } + + @Override + public void close() { + // 无底层资源需要关闭。 + } + }); + + @Override + public MediaType contentType() { + return MediaType.parse("text/plain"); + } + + @Override + public long contentLength() { + return 19; + } + + @Override + public BufferedSource source() { + return source; + } + }; + return responseBuilder + .code(500) + .message("Internal Server Error") + .body(trackingBody) + .build(); + }) + .build(); + MineruProperties properties = defaultProperties(); + MineruClient client = new MineruClient(properties, httpClient, new MineruMapper(properties)); + + try { + client.submit(buildRequest()); + Assert.fail("Expected MinerU server error"); + } catch (DocumentParseException expected) { + Assert.assertTrue(expected.getMessage().contains("status=500")); + } + + Assert.assertEquals(2, requestCount.get()); + Assert.assertFalse("5xx response body should not be read", errorBodyRead.get()); + } + private ParseRequest buildRequest() { ParseRequest request = new ParseRequest(); request.addFile(ParseFile.of("demo.pptx", "ppt".getBytes(StandardCharsets.UTF_8))); diff --git a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/mineru/CommonMineruDocumentProperties.java b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/mineru/CommonMineruDocumentProperties.java index 1e26e56..41f9537 100644 --- a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/mineru/CommonMineruDocumentProperties.java +++ b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/mineru/CommonMineruDocumentProperties.java @@ -19,6 +19,7 @@ public class CommonMineruDocumentProperties { private Integer connectTimeoutMs = 3000; private Integer readTimeoutMs = 600000; private Integer writeTimeoutMs = 600000; + private Integer submitTimeoutMs = 120000; private Integer pollIntervalMs = 1000; private Integer resultTimeoutMs = 1800000; private String defaultBackend = "vlm-http-client"; @@ -59,6 +60,24 @@ public class CommonMineruDocumentProperties { this.writeTimeoutMs = writeTimeoutMs; } + /** + * 获取异步任务提交总超时时间。 + * + * @return 提交总超时时间,单位毫秒 + */ + public Integer getSubmitTimeoutMs() { + return submitTimeoutMs; + } + + /** + * 设置异步任务提交总超时时间。 + * + * @param submitTimeoutMs 提交总超时时间,单位毫秒 + */ + public void setSubmitTimeoutMs(Integer submitTimeoutMs) { + this.submitTimeoutMs = submitTimeoutMs; + } + public Integer getPollIntervalMs() { return pollIntervalMs; } diff --git a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pdf/mineru/MineruPdfAutoConfiguration.java b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pdf/mineru/MineruPdfAutoConfiguration.java index 9cdcc74..be723e6 100644 --- a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pdf/mineru/MineruPdfAutoConfiguration.java +++ b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pdf/mineru/MineruPdfAutoConfiguration.java @@ -105,6 +105,7 @@ public class MineruPdfAutoConfiguration { mineruProperties.setConnectTimeoutMs(useCommon ? commonProperties.getConnectTimeoutMs() : null); mineruProperties.setReadTimeoutMs(useCommon ? commonProperties.getReadTimeoutMs() : null); mineruProperties.setWriteTimeoutMs(useCommon ? commonProperties.getWriteTimeoutMs() : null); + mineruProperties.setSubmitTimeoutMs(useCommon ? commonProperties.getSubmitTimeoutMs() : null); mineruProperties.setPollIntervalMs(useCommon ? commonProperties.getPollIntervalMs() : null); mineruProperties.setResultTimeoutMs(useCommon ? commonProperties.getResultTimeoutMs() : null); mineruProperties.setDefaultBackend(useCommon ? commonProperties.getDefaultBackend() : null); diff --git a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pptx/MineruPptxAutoConfiguration.java b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pptx/MineruPptxAutoConfiguration.java index c367776..a2fbab1 100644 --- a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pptx/MineruPptxAutoConfiguration.java +++ b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/pptx/MineruPptxAutoConfiguration.java @@ -53,6 +53,7 @@ public class MineruPptxAutoConfiguration { mineruProperties.setConnectTimeoutMs(properties.getConnectTimeoutMs()); mineruProperties.setReadTimeoutMs(properties.getReadTimeoutMs()); mineruProperties.setWriteTimeoutMs(properties.getWriteTimeoutMs()); + mineruProperties.setSubmitTimeoutMs(properties.getSubmitTimeoutMs()); mineruProperties.setPollIntervalMs(properties.getPollIntervalMs()); mineruProperties.setResultTimeoutMs(properties.getResultTimeoutMs()); mineruProperties.setDefaultBackend(properties.getDefaultBackend()); diff --git a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/xlsx/MineruXlsxAutoConfiguration.java b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/xlsx/MineruXlsxAutoConfiguration.java index 848e7a2..e580fa4 100644 --- a/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/xlsx/MineruXlsxAutoConfiguration.java +++ b/easy-agents-spring-boot-starter/src/main/java/com/easyagents/spring/boot/document/xlsx/MineruXlsxAutoConfiguration.java @@ -53,6 +53,7 @@ public class MineruXlsxAutoConfiguration { mineruProperties.setConnectTimeoutMs(properties.getConnectTimeoutMs()); mineruProperties.setReadTimeoutMs(properties.getReadTimeoutMs()); mineruProperties.setWriteTimeoutMs(properties.getWriteTimeoutMs()); + mineruProperties.setSubmitTimeoutMs(properties.getSubmitTimeoutMs()); mineruProperties.setPollIntervalMs(properties.getPollIntervalMs()); mineruProperties.setResultTimeoutMs(properties.getResultTimeoutMs()); mineruProperties.setDefaultBackend(properties.getDefaultBackend()); From 5fd4d845af4b4bfaddad326a3ea6bbca22090dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 3 Aug 2026 14:55:32 +0800 Subject: [PATCH 21/33] =?UTF-8?q?fix:=20=E5=85=BC=E5=AE=B9=20Milvus=20?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E4=B8=B2=E4=B8=BB=E9=94=AE=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 删除前统一规范化主键值 - 补充字符串主键回归测试 --- .../store/milvus/MilvusPrimaryKeySupport.java | 39 +++++++++++++++++ .../store/milvus/MilvusVectorStore.java | 6 ++- .../milvus/MilvusPrimaryKeySupportTest.java | 42 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusPrimaryKeySupport.java create mode 100644 easy-agents-store/easy-agents-store-milvus/src/test/java/com/easyagents/store/milvus/MilvusPrimaryKeySupportTest.java diff --git a/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusPrimaryKeySupport.java b/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusPrimaryKeySupport.java new file mode 100644 index 0000000..6d1ff1c --- /dev/null +++ b/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusPrimaryKeySupport.java @@ -0,0 +1,39 @@ +package com.easyagents.store.milvus; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +/** + * Milvus VarChar 主键转换工具。 + */ +final class MilvusPrimaryKeySupport { + + /** + * 禁止实例化无状态工具类。 + */ + private MilvusPrimaryKeySupport() { + } + + /** + * 将删除主键归一化为 Milvus VarChar 主键值。 + * + *

Milvus 向量存储创建的集合固定使用 VarChar 类型的 {@code id} 主键。 + * Java SDK 会根据传入值类型生成删除表达式,因此数值对象必须先转为字符串。

+ * + * @param ids 调用方提供的主键集合 + * @return 可直接传给 Milvus Java SDK 的字符串主键列表 + * @throws NullPointerException 主键集合中包含空值时抛出 + */ + static List normalize(Collection ids) { + List normalizedIds = new ArrayList(ids.size()); + for (Object id : ids) { + normalizedIds.add(Objects.requireNonNull( + id, + "Milvus primary key must not be null" + ).toString()); + } + return normalizedIds; + } +} diff --git a/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusVectorStore.java b/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusVectorStore.java index aed63f1..efee8fc 100644 --- a/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusVectorStore.java +++ b/easy-agents-store/easy-agents-store-milvus/src/main/java/com/easyagents/store/milvus/MilvusVectorStore.java @@ -157,12 +157,14 @@ public class MilvusVectorStore extends DocumentStore implements AutoCloseable { } DeleteReq deleteReq = builder .collectionName(collectionName) - .ids(new ArrayList(ids)) + .ids(MilvusPrimaryKeySupport.normalize(ids)) .build(); client.delete(deleteReq); return StoreResult.success(); } catch (Exception e) { - return StoreResult.fail(); + LOG.error("Milvus delete failed. collection={}, message={}", + collectionName, e.getMessage(), e); + return StoreResult.fail(e.getMessage()); } } diff --git a/easy-agents-store/easy-agents-store-milvus/src/test/java/com/easyagents/store/milvus/MilvusPrimaryKeySupportTest.java b/easy-agents-store/easy-agents-store-milvus/src/test/java/com/easyagents/store/milvus/MilvusPrimaryKeySupportTest.java new file mode 100644 index 0000000..8efa250 --- /dev/null +++ b/easy-agents-store/easy-agents-store-milvus/src/test/java/com/easyagents/store/milvus/MilvusPrimaryKeySupportTest.java @@ -0,0 +1,42 @@ +package com.easyagents.store.milvus; + +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.List; + +/** + * {@link MilvusPrimaryKeySupport} 主键归一化测试。 + */ +public class MilvusPrimaryKeySupportTest { + + /** + * 验证数值和字符串主键都按 VarChar 类型传给 Milvus。 + */ + @Test + public void shouldNormalizeDeleteIdsAsStrings() { + List ids = MilvusPrimaryKeySupport.normalize(Arrays.asList( + new BigInteger("105257799143000107"), + 42L, + "faq-1" + )); + + Assert.assertEquals( + Arrays.asList("105257799143000107", "42", "faq-1"), + ids + ); + for (Object id : ids) { + Assert.assertTrue(id instanceof String); + } + } + + /** + * 验证空主键会被明确拒绝,避免生成无效删除表达式。 + */ + @Test(expected = NullPointerException.class) + public void shouldRejectNullDeleteId() { + MilvusPrimaryKeySupport.normalize(Arrays.asList("1", null)); + } +} From bdb69a2250dd50bbd7dad43131a19e1e102835a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Mon, 3 Aug 2026 16:58:11 +0800 Subject: [PATCH 22/33] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E8=8A=82=E7=82=B9=20main=20=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 按节点参数顺序向 JavaScript main 传入独立参数值 - 保留历史单对象参数与显式 _result 输出兼容 - 补充参数模式解析和运行时回归测试 --- .../code/impl/JavascriptRuntimeEngine.java | 73 +++++++++++++++++-- .../easyagents/flow/core/node/CodeNode.java | 27 +++++++ .../flow/core/parser/impl/CodeNodeParser.java | 1 + .../test/JavascriptRuntimeEngineTest.java | 64 ++++++++++++++++ 4 files changed, 158 insertions(+), 7 deletions(-) diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java index 76a8a69..4931d48 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/code/impl/JavascriptRuntimeEngine.java @@ -19,6 +19,7 @@ import com.alibaba.fastjson.JSONObject; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.ChainState; import com.easyagents.flow.core.chain.NodeState; +import com.easyagents.flow.core.chain.Parameter; import com.easyagents.flow.core.code.CodeRuntimeEngine; import com.easyagents.flow.core.node.CodeNode; import com.easyagents.flow.core.util.graalvm.JsInteropUtils; @@ -70,6 +71,10 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { .build(); private static final Source RESULT_INIT_SOURCE = Source.create("js", "var _result = {};"); + private static final Source STRICT_EQUAL_SOURCE = + Source.create( + "js", + "(left, right) => left === right;"); private static final List> SOURCE_CACHE_SEGMENTS = createSourceCacheSegments(); private static final ScheduledThreadPoolExecutor @@ -159,6 +164,7 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { // 在 JS 中创建 _result 对象 context.eval(RESULT_INIT_SOURCE); + Value initialResult = bindings.getMember("_result"); // 注入 _chain 和 _context bindings.putMember("_chain", chain); @@ -167,7 +173,18 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { // 执行用户脚本 context.eval(source(code, node)); - return resolveResult(context, bindings, parameterValues); + boolean legacyResultAssigned = + !context.eval(STRICT_EQUAL_SOURCE) + .execute( + initialResult, + bindings.getMember("_result")) + .asBoolean(); + return resolveResult( + context, + bindings, + node, + parameterValues, + legacyResultAssigned); } catch (PolyglotException e) { throw executionException(e, timedOut.get()); } catch (Exception e) { @@ -389,16 +406,20 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { * * @param context JavaScript 执行上下文 * @param bindings JavaScript 全局绑定 + * @param node 当前代码节点 * @param parameterValues 当前节点已解析的输入参数 + * @param legacyResultAssigned 用户代码是否重新赋值 _result * @return 代码节点输出 * @throws IllegalArgumentException main 返回值不是对象时抛出 */ private Map resolveResult(Context context, Value bindings, - Map parameterValues) { + CodeNode node, + Map parameterValues, + boolean legacyResultAssigned) { JSONObject legacyResult = GraalvmToFastJSONUtils.toJSONObject( bindings.getMember("_result")); - if (!legacyResult.isEmpty()) { + if (legacyResultAssigned || !legacyResult.isEmpty()) { return legacyResult; } @@ -407,11 +428,8 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { return legacyResult; } - Map mainInput = parameterValues == null - ? Collections.emptyMap() - : parameterValues; Value mainResult = mainFunction.execute( - JsInteropUtils.wrapJavaValueForJS(context, mainInput)); + resolveMainArguments(context, node, parameterValues)); Object convertedResult = GraalvmToFastJSONUtils.toFastJsonValue(mainResult); if (!(convertedResult instanceof JSONObject)) { @@ -421,4 +439,45 @@ public class JavascriptRuntimeEngine implements CodeRuntimeEngine { return (JSONObject) convertedResult; } + /** + * 根据节点契约构建 main 自动调用参数。 + * + * @param context JavaScript 执行上下文 + * @param node 当前代码节点 + * @param parameterValues 当前节点已解析的输入参数 + * @return GraalVM main 调用参数 + */ + private Object[] resolveMainArguments( + Context context, + CodeNode node, + Map parameterValues) { + if (node != null + && CodeNode.MAIN_ARGS_MODE_NAMED.equals( + node.getMainArgsMode())) { + List parameters = node.getParameters(); + if (parameters == null || parameters.isEmpty()) { + return new Object[0]; + } + Object[] arguments = new Object[parameters.size()]; + for (int index = 0; index < parameters.size(); index++) { + String name = parameters.get(index).getName(); + Object value = parameterValues == null + ? null + : parameterValues.get(name); + arguments[index] = + JsInteropUtils.wrapJavaValueForJS( + context, value); + } + return arguments; + } + + Map mainInput = parameterValues == null + ? Collections.emptyMap() + : parameterValues; + return new Object[]{ + JsInteropUtils.wrapJavaValueForJS( + context, mainInput) + }; + } + } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java index e49d839..329aa07 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/CodeNode.java @@ -29,8 +29,17 @@ import java.util.Map; public class CodeNode extends BaseNode { private static final long serialVersionUID = 1L; + /** + * main 按节点参数顺序接收独立参数。 + */ + public static final String MAIN_ARGS_MODE_NAMED = "named"; + protected String engine; protected String code; + /** + * main 自动调用参数模式;空值沿用历史单对象参数。 + */ + protected String mainArgsMode; public String getEngine() { return engine; @@ -48,6 +57,24 @@ public class CodeNode extends BaseNode { this.code = code; } + /** + * 获取 main 自动调用参数模式。 + * + * @return 参数模式;空值表示历史单对象参数模式 + */ + public String getMainArgsMode() { + return mainArgsMode; + } + + /** + * 设置 main 自动调用参数模式。 + * + * @param mainArgsMode 参数模式 + */ + public void setMainArgsMode(String mainArgsMode) { + this.mainArgsMode = mainArgsMode; + } + @Override public Map execute(Chain chain) { if (StringUtil.noText(code)) { diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/CodeNodeParser.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/CodeNodeParser.java index e2affb3..41eeb43 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/CodeNodeParser.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/parser/impl/CodeNodeParser.java @@ -27,6 +27,7 @@ public class CodeNodeParser extends BaseNodeParser { CodeNode codeNode = new CodeNode(); codeNode.setEngine(engine); codeNode.setCode(data.getString("code")); + codeNode.setMainArgsMode(data.getString("mainArgsMode")); return codeNode; } } diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java index 982b534..7514501 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/JavascriptRuntimeEngineTest.java @@ -1,5 +1,6 @@ package com.easyagents.flow.core.test; +import com.alibaba.fastjson.JSONObject; import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.ChainDefinition; import com.easyagents.flow.core.chain.ChainState; @@ -9,6 +10,7 @@ import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; import com.easyagents.flow.core.code.impl.JavascriptExecutionException; import com.easyagents.flow.core.code.impl.JavascriptRuntimeEngine; import com.easyagents.flow.core.node.CodeNode; +import com.easyagents.flow.core.parser.impl.CodeNodeParser; import org.junit.Assert; import org.junit.Test; @@ -57,6 +59,46 @@ public class JavascriptRuntimeEngineTest { Assert.assertEquals(4L, result.get("totalLength")); } + /** + * 验证新节点按配置顺序把输入值传给 main 的独立形参。 + */ + @Test + public void shouldInvokeMainWithNamedArguments() { + CodeNode node = codeNode( + String.join("\n", + "function main(data1, data2) {", + " return { joined: data1 + data2 };", + "}"), + "data1", + "data2"); + node.setMainArgsMode(CodeNode.MAIN_ARGS_MODE_NAMED); + Chain chain = chain(Map.of( + "data1", "河北", + "data2", "分行")); + + Map result = node.execute(chain); + + Assert.assertEquals("河北分行", result.get("joined")); + } + + /** + * 验证解析器保留设计器声明的 main 参数模式。 + */ + @Test + public void shouldParseMainArgsMode() { + JSONObject data = new JSONObject(); + data.put("engine", "js"); + data.put("code", "function main(data) { return { data }; }"); + data.put("mainArgsMode", CodeNode.MAIN_ARGS_MODE_NAMED); + + CodeNode node = new CodeNodeParser().doParse( + new JSONObject(), data, new JSONObject()); + + Assert.assertEquals( + CodeNode.MAIN_ARGS_MODE_NAMED, + node.getMainArgsMode()); + } + /** * 验证已有 _result 输出优先,避免自动调用 main 改变历史流程。 */ @@ -75,6 +117,28 @@ public class JavascriptRuntimeEngineTest { Assert.assertEquals("legacy", result.get("answer")); } + /** + * 验证显式调用返回空对象时不会再次自动调用 main。 + */ + @Test + public void shouldKeepEmptyExplicitResultWithoutInvokingMainAgain() { + CodeNode node = codeNode( + String.join("\n", + "function main(data) {", + " if (typeof data !== 'string') {", + " throw new Error('main invoked twice');", + " }", + " return {};", + "}", + "_result = main(data);"), + "data"); + Chain chain = chain(Map.of("data", "hello")); + + Map result = node.execute(chain); + + Assert.assertTrue(result.isEmpty()); + } + /** * 验证 main 返回非对象时给出明确错误。 */ From f13e24751ae512d10dc323f8c76176a59d163239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 7 Aug 2026 12:38:09 +0800 Subject: [PATCH 23/33] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E7=B4=A2=E5=BC=95=E5=B9=B6=E5=8F=91=E4=B8=8E?= =?UTF-8?q?=E5=90=91=E9=87=8F=E5=8C=96=E8=BE=B9=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 复用 Lucene 和 Elasticsearch 客户端并支持有界批量写入与删除 - 记录脱敏后的 Embedding 失败请求与完整响应 - 为 BGE-M3 分块统一增加上下文硬上限 --- .../openai/OpenAIEmbeddingModel.java | 81 ++++- .../openai/OpenAIEmbeddingModelTest.java | 43 +++ .../easyagents/rag/core/BgeM3ChunkSafety.java | 230 ++++++++++++ .../com/easyagents/rag/core/RagDefaults.java | 14 + .../chunk/RagSplitStrategyRegistry.java | 246 ++++++++++--- .../rag/ingestion/model/StrategyConfig.java | 24 ++ .../ingestion/RagIngestionPipelineTest.java | 152 ++++++++ .../easyagents/engine/es/ElasticSearcher.java | 330 +++++++++++++----- .../es/ElasticSearcherQueryBuilderTest.java | 183 ++++++++-- .../search/engine/lucene/LuceneSearcher.java | 204 +++++++---- .../engine/lucene/LuceneSearcherTest.java | 145 +++++++- .../engine/service/DocumentSearcher.java | 83 +++++ 12 files changed, 1472 insertions(+), 263 deletions(-) create mode 100644 easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/BgeM3ChunkSafety.java diff --git a/easy-agents-embedding/easy-agents-embedding-openai/src/main/java/com/easyagents/embedding/openai/OpenAIEmbeddingModel.java b/easy-agents-embedding/easy-agents-embedding-openai/src/main/java/com/easyagents/embedding/openai/OpenAIEmbeddingModel.java index afbc8c2..cd8e0ed 100644 --- a/easy-agents-embedding/easy-agents-embedding-openai/src/main/java/com/easyagents/embedding/openai/OpenAIEmbeddingModel.java +++ b/easy-agents-embedding/easy-agents-embedding-openai/src/main/java/com/easyagents/embedding/openai/OpenAIEmbeddingModel.java @@ -11,13 +11,19 @@ import com.easyagents.core.store.VectorData; import com.easyagents.core.util.JSONUtil; import com.easyagents.core.util.Maps; import com.easyagents.core.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; public class OpenAIEmbeddingModel extends BaseEmbeddingModel { + private static final Logger LOG = LoggerFactory.getLogger(OpenAIEmbeddingModel.class); + private static final String REDACTED_HEADER_VALUE = "[REDACTED]"; + private HttpClient httpClient = new HttpClient(); public OpenAIEmbeddingModel(OpenAIEmbeddingConfig config) { @@ -40,27 +46,45 @@ public class OpenAIEmbeddingModel extends BaseEmbeddingModel sanitizeHeadersForLogging(Map headers) { + Map sanitizedHeaders = new LinkedHashMap<>(); + if (headers == null) { + return sanitizedHeaders; + } + headers.forEach((name, value) -> sanitizedHeaders.put( + name, + "Authorization".equalsIgnoreCase(name) ? REDACTED_HEADER_VALUE : value + )); + return sanitizedHeaders; + } } diff --git a/easy-agents-embedding/easy-agents-embedding-openai/src/test/java/com/easyagents/embedding/openai/OpenAIEmbeddingModelTest.java b/easy-agents-embedding/easy-agents-embedding-openai/src/test/java/com/easyagents/embedding/openai/OpenAIEmbeddingModelTest.java index 1017c08..edf3a5f 100644 --- a/easy-agents-embedding/easy-agents-embedding-openai/src/test/java/com/easyagents/embedding/openai/OpenAIEmbeddingModelTest.java +++ b/easy-agents-embedding/easy-agents-embedding-openai/src/test/java/com/easyagents/embedding/openai/OpenAIEmbeddingModelTest.java @@ -7,6 +7,7 @@ import com.easyagents.core.model.exception.ModelException; import org.junit.Assert; import org.junit.Test; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -68,4 +69,46 @@ public class OpenAIEmbeddingModelTest { Assert.assertTrue(exception.getMessage().contains("data[0].embedding")); } + + /** + * Verifies that invalid JSON responses are reported as embedding response parsing failures. + */ + @Test + public void shouldThrowModelExceptionWhenResponseIsInvalidJson() { + OpenAIEmbeddingConfig config = new OpenAIEmbeddingConfig(); + config.setProvider("test-provider"); + config.setModel("BAAI/bge-m3"); + config.setApiKey("test-key"); + OpenAIEmbeddingModel model = new OpenAIEmbeddingModel(config); + model.setHttpClient(new HttpClient() { + @Override + public String post(String url, Map headers, String payload) { + return "not-json"; + } + }); + + ModelException exception = Assert.assertThrows( + ModelException.class, + () -> model.embed(Document.of("hello")) + ); + + Assert.assertEquals("Failed to parse embedding response.", exception.getMessage()); + Assert.assertNotNull(exception.getCause()); + } + + /** + * Verifies that diagnostic headers retain ordinary values without exposing credentials. + */ + @Test + public void shouldRedactAuthorizationHeaderForFailureLogging() { + Map headers = new LinkedHashMap<>(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Bearer test-key"); + + Map sanitizedHeaders = OpenAIEmbeddingModel.sanitizeHeadersForLogging(headers); + + Assert.assertEquals("application/json", sanitizedHeaders.get("Content-Type")); + Assert.assertEquals("[REDACTED]", sanitizedHeaders.get("Authorization")); + Assert.assertEquals("Bearer test-key", headers.get("Authorization")); + } } diff --git a/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/BgeM3ChunkSafety.java b/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/BgeM3ChunkSafety.java new file mode 100644 index 0000000..4865ae5 --- /dev/null +++ b/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/BgeM3ChunkSafety.java @@ -0,0 +1,230 @@ +package com.easyagents.rag.core; + +import java.text.Normalizer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Provides conservative BGE-M3 token estimation and hard-limit text splitting. + * + *

BGE-M3 uses an XLM-R SentencePiece tokenizer. Loading the model vocabulary + * during ingestion would introduce runtime model downloads, so this guard counts + * NFKC-normalized Unicode code points. The estimate is intentionally conservative + * for ordinary Chinese and mixed-language knowledge documents.

+ */ +public final class BgeM3ChunkSafety { + + private static final double PREFERRED_BOUNDARY_MIN_RATIO = 0.8D; + + private BgeM3ChunkSafety() { + } + + /** + * Estimates the content token count after tokenizer-compatible normalization. + * + * @param content chunk content + * @return conservative token estimate, or zero for empty content + */ + public static int estimateContentTokens(String content) { + if (content == null || content.isEmpty()) { + return 0; + } + String normalized = Normalizer.normalize(content, Normalizer.Form.NFKC); + return normalized.codePointCount(0, normalized.length()); + } + + /** + * Checks whether content stays within the shared BGE-M3 embedding budget. + * + * @param content chunk content + * @return true when the content can be sent to BGE-M3 safely + */ + public static boolean isWithinHardLimit(String content) { + return estimateContentTokens(content) + <= RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT; + } + + /** + * Splits content using the shared BGE-M3 hard token limit. + * + * @param content source content + * @return ordered, non-overlapping source ranges + */ + public static List splitToHardLimit(String content) { + return splitToTokenLimit(content, RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT); + } + + /** + * Splits content to an explicit conservative token limit. + * + * @param content source content + * @param maxTokens maximum estimated tokens per part + * @return ordered, non-overlapping source ranges + * @throws IllegalArgumentException when maxTokens is not positive + */ + public static List splitToTokenLimit(String content, int maxTokens) { + if (maxTokens <= 0) { + throw new IllegalArgumentException("maxTokens must be positive"); + } + if (content == null || content.isEmpty()) { + return Collections.emptyList(); + } + if (estimateContentTokens(content) <= maxTokens) { + return Collections.singletonList(new ChunkRange(0, content.length())); + } + + List parts = new ArrayList(); + int start = 0; + while (start < content.length()) { + int end = findSplitEnd(content, start, maxTokens); + if (end <= start) { + throw new IllegalStateException("BGE-M3 hard split did not advance the text cursor"); + } + parts.add(new ChunkRange(start, end)); + start = end; + } + return parts; + } + + /** + * Finds a safe source boundary while preferring nearby sentence endings. + * + * @param content complete source content + * @param start current part start + * @param maxTokens maximum estimated tokens + * @return exclusive end offset + */ + private static int findSplitEnd(String content, int start, int maxTokens) { + int cursor = start; + int currentTokens = 0; + int preferredEnd = -1; + int preferredBoundaryThreshold = Math.max( + 1, (int) Math.floor(maxTokens * PREFERRED_BOUNDARY_MIN_RATIO)); + while (cursor < content.length()) { + int codePoint = content.codePointAt(cursor); + int next = cursor + Character.charCount(codePoint); + int nextTokens = currentTokens + estimateCodePointTokens(codePoint); + if (nextTokens > maxTokens) { + break; + } + cursor = next; + currentTokens = nextTokens; + if (currentTokens >= preferredBoundaryThreshold + && isPreferredBoundary(codePoint)) { + preferredEnd = cursor; + } + } + int candidateEnd = preferredEnd > start ? preferredEnd : cursor; + if (candidateEnd <= start) { + candidateEnd = start + Character.charCount(content.codePointAt(start)); + } + return fitWithinTokenLimit(content, start, candidateEnd, maxTokens); + } + + /** + * Shrinks a candidate when Unicode normalization expands its token estimate. + * + * @param content complete source content + * @param start part start + * @param candidateEnd proposed exclusive end + * @param maxTokens maximum estimated tokens + * @return exclusive end within the requested token limit + */ + private static int fitWithinTokenLimit( + String content, + int start, + int candidateEnd, + int maxTokens) { + if (estimateContentTokens(content.substring(start, candidateEnd)) + <= maxTokens) { + return candidateEnd; + } + int low = 1; + int high = content.codePointCount(start, candidateEnd); + while (low < high) { + int middle = low + (high - low + 1) / 2; + int end = content.offsetByCodePoints(start, middle); + if (estimateContentTokens(content.substring(start, end)) <= maxTokens) { + low = middle; + } else { + high = middle - 1; + } + } + return content.offsetByCodePoints(start, low); + } + + /** + * Estimates the normalized token cost of one source code point. + * + * @param codePoint source Unicode code point + * @return conservative token estimate for the code point + */ + private static int estimateCodePointTokens(int codePoint) { + if (codePoint >= 0 && codePoint <= 0x7F) { + return 1; + } + String source = new String(Character.toChars(codePoint)); + String normalized = Normalizer.normalize(source, Normalizer.Form.NFKC); + return Math.max(1, normalized.codePointCount(0, normalized.length())); + } + + /** + * Determines whether a code point is a suitable semantic boundary. + * + * @param codePoint current Unicode code point + * @return true for sentence, paragraph, or whitespace boundaries + */ + private static boolean isPreferredBoundary(int codePoint) { + return Character.isWhitespace(codePoint) + || codePoint == '。' + || codePoint == '!' + || codePoint == '?' + || codePoint == ';' + || codePoint == '.' + || codePoint == '!' + || codePoint == '?' + || codePoint == ';'; + } + + /** + * Immutable UTF-16 source range returned by the hard splitter. + */ + public static final class ChunkRange { + + private final int start; + private final int end; + + /** + * Creates a source range. + * + * @param start inclusive UTF-16 offset + * @param end exclusive UTF-16 offset + */ + public ChunkRange(int start, int end) { + if (start < 0 || end < start) { + throw new IllegalArgumentException("Invalid chunk range"); + } + this.start = start; + this.end = end; + } + + /** + * Returns the inclusive start offset. + * + * @return inclusive UTF-16 offset + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive end offset. + * + * @return exclusive UTF-16 offset + */ + public int getEnd() { + return end; + } + } +} diff --git a/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/RagDefaults.java b/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/RagDefaults.java index a78c377..7a2aaba 100644 --- a/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/RagDefaults.java +++ b/easy-agents-rag/easy-agents-rag-core/src/main/java/com/easyagents/rag/core/RagDefaults.java @@ -1,12 +1,26 @@ package com.easyagents.rag.core; +/** + * RAG ingestion default values and safety limits. + */ public final class RagDefaults { private RagDefaults() { } + /** Default strategy chunk size. */ public static final int CHUNK_SIZE = 512; + /** Default overlap size. */ public static final int OVERLAP_SIZE = 128; + /** Default Markdown heading level. */ public static final int MD_SPLITTER_LEVEL = 2; + /** Default spreadsheet rows per chunk. */ public static final int ROWS_PER_CHUNK = 1; + /** BGE-M3 maximum sequence length. */ + public static final int BGE_M3_MAX_SEQUENCE_TOKENS = 8192; + /** Reserved budget for model special tokens and tokenizer estimation differences. */ + public static final int BGE_M3_RESERVED_TOKENS = 128; + /** Hard upper token estimate for chunk content sent to BGE-M3. */ + public static final int BGE_M3_HARD_CHUNK_TOKEN_LIMIT = + BGE_M3_MAX_SEQUENCE_TOKENS - BGE_M3_RESERVED_TOKENS; } diff --git a/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/chunk/RagSplitStrategyRegistry.java b/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/chunk/RagSplitStrategyRegistry.java index d915857..b46bcc6 100644 --- a/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/chunk/RagSplitStrategyRegistry.java +++ b/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/chunk/RagSplitStrategyRegistry.java @@ -21,19 +21,19 @@ public class RagSplitStrategyRegistry { strategyCode = analysisResult.getRecommendedStrategyCode(); } String normalizedContent = analysisResult.getNormalizedContent(); + List chunks; if (RagStrategyCodes.MARKDOWN_SECTION.equals(strategyCode)) { - return buildMarkdownChunks(normalizedContent, strategyConfig); + chunks = buildMarkdownChunks(normalizedContent, strategyConfig); + } else if (RagStrategyCodes.OUTLINE_SECTION.equals(strategyCode)) { + chunks = buildOutlineChunks(normalizedContent, strategyConfig); + } else if (RagStrategyCodes.QA_PAIR.equals(strategyCode)) { + chunks = buildQaChunks(normalizedContent, strategyConfig); + } else if (RagStrategyCodes.CUSTOM_REGEX.equals(strategyCode)) { + chunks = buildRegexChunks(normalizedContent, strategyConfig); + } else { + chunks = buildParagraphChunks(normalizedContent, strategyConfig); } - if (RagStrategyCodes.OUTLINE_SECTION.equals(strategyCode)) { - return buildOutlineChunks(normalizedContent, strategyConfig); - } - if (RagStrategyCodes.QA_PAIR.equals(strategyCode)) { - return buildQaChunks(normalizedContent, strategyConfig); - } - if (RagStrategyCodes.CUSTOM_REGEX.equals(strategyCode)) { - return buildRegexChunks(normalizedContent, strategyConfig); - } - return buildParagraphChunks(normalizedContent, strategyConfig); + return postProcess(enforceBgeM3HardLimit(chunks)); } private List buildMarkdownChunks(String content, StrategyConfig strategyConfig) { @@ -143,7 +143,7 @@ public class RagSplitStrategyRegistry { )); } } - return postProcess(result); + return result; } private List buildQaChunks(String content, StrategyConfig strategyConfig) { @@ -162,7 +162,14 @@ public class RagSplitStrategyRegistry { Matcher questionMatcher = QUESTION_PREFIX.matcher(line); Matcher answerMatcher = ANSWER_PREFIX.matcher(line); if (questionMatcher.matches()) { - qaIndex = flushQaChunk(result, currentQuestion, questionSlices, answerSlices, qaIndex, strategyConfig); + qaIndex = flushQaChunk( + result, + content, + currentQuestion, + questionSlices, + answerSlices, + qaIndex, + strategyConfig); currentQuestion = questionMatcher.group(2).trim(); questionSlices = new ArrayList(); answerSlices = new ArrayList(); @@ -187,11 +194,19 @@ public class RagSplitStrategyRegistry { questionSlices.add(lineSlice); } } - flushQaChunk(result, currentQuestion, questionSlices, answerSlices, qaIndex, strategyConfig); - return postProcess(result); + flushQaChunk( + result, + content, + currentQuestion, + questionSlices, + answerSlices, + qaIndex, + strategyConfig); + return result; } private int flushQaChunk(List result, + String content, String currentQuestion, List questionSlices, List answerSlices, @@ -203,27 +218,59 @@ public class RagSplitStrategyRegistry { if (answerSlices == null || answerSlices.isEmpty()) { return qaIndex; } - String question = joinLineSlices(questionSlices); - String answer = joinLineSlices(answerSlices); - String baseContent = "问题:" + question + "\n答案:" + answer; - List subContents = baseContent.length() > safeChunkSize(strategyConfig) - ? splitLongContent(baseContent, strategyConfig.getChunkSize()) - : Collections.singletonList(baseContent); - int total = subContents.size(); - List sourceRanges = buildQaSourceRanges(questionSlices, answerSlices); - for (int i = 0; i < subContents.size(); i++) { + TextRange questionRange = mergeLineSlices(questionSlices); + TextRange rawAnswerRange = mergeLineSlices(answerSlices); + TextRange answerRange = rawAnswerRange == null + ? null + : trimRange(content, rawAnswerRange.start, rawAnswerRange.end); + if (questionRange == null || answerRange == null) { + return qaIndex; + } + + String question = currentQuestion.trim(); + String answer = content.substring(answerRange.start, answerRange.end); + String qaPrefix = "问题:" + question + "\n答案:"; + int requestedTokens = Math.min( + safeChunkSize(strategyConfig), + RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT); + int prefixTokens = BgeM3ChunkSafety.estimateContentTokens(qaPrefix); + boolean includePrefix = prefixTokens + < RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT; + int answerTokenLimit = includePrefix + ? Math.max(1, Math.max(requestedTokens, prefixTokens + 1) - prefixTokens) + : RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT; + List answerPartRanges = new ArrayList(); + for (BgeM3ChunkSafety.ChunkRange answerPart + : BgeM3ChunkSafety.splitToTokenLimit(answer, answerTokenLimit)) { + TextRange absoluteAnswerRange = trimRange( + content, + answerRange.start + answerPart.getStart(), + answerRange.start + answerPart.getEnd()); + if (absoluteAnswerRange != null) { + answerPartRanges.add(absoluteAnswerRange); + } + } + int total = answerPartRanges.size(); + for (int i = 0; i < answerPartRanges.size(); i++) { + TextRange absoluteAnswerRange = answerPartRanges.get(i); + String answerFragment = content.substring( + absoluteAnswerRange.start, + absoluteAnswerRange.end); + List sourceRanges = new ArrayList(); + sourceRanges.add(questionRange); + sourceRanges.add(absoluteAnswerRange); RagChunk chunk = createChunk( RagChunkTypes.QA_PAIR, "Q" + qaIndex + " " + question, Collections.emptyList(), - subContents.get(i), + includePrefix ? qaPrefix + answerFragment : answerFragment, result.size() + 1, i + 1, total, sourceRanges ); chunk.setQuestion(question); - chunk.setAnswer(answer); + chunk.setAnswer(answerFragment); chunk.getOptions().put(RagMetadataKeys.QA_GROUP_ID, "qa-" + qaIndex); result.add(chunk); } @@ -254,7 +301,7 @@ public class RagSplitStrategyRegistry { )); index++; } - return postProcess(result); + return result; } private List buildRegexChunks(String content, StrategyConfig strategyConfig) { @@ -264,42 +311,129 @@ public class RagSplitStrategyRegistry { Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(content); int segmentStart = 0; + boolean retainRegexMatch = Boolean.TRUE.equals( + strategyConfig.getRetainRegexMatch()); while (matcher.find()) { index = addRegexChunk(result, content, segmentStart, matcher.start(), index); - segmentStart = matcher.end(); + // 保留时将匹配内容归入下一分块;默认继续从匹配结束位置开始。 + segmentStart = retainRegexMatch ? matcher.start() : matcher.end(); } addRegexChunk(result, content, segmentStart, content.length(), index); - return postProcess(result); + return result; } private List splitLongContent(String content, Integer chunkSize) { - int size = chunkSize == null || chunkSize.intValue() <= 0 ? RagDefaults.CHUNK_SIZE : chunkSize.intValue(); - String[] paragraphs = content.split("\\n\\s*\\n"); + int size = chunkSize == null || chunkSize.intValue() <= 0 + ? RagDefaults.CHUNK_SIZE + : Math.min(chunkSize.intValue(), RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT); List parts = new ArrayList(); - StringBuilder current = new StringBuilder(); - for (String paragraph : paragraphs) { - String text = paragraph.trim(); - if (!StringUtil.hasText(text)) { - continue; + for (BgeM3ChunkSafety.ChunkRange range + : BgeM3ChunkSafety.splitToTokenLimit(content, size)) { + String part = content.substring(range.getStart(), range.getEnd()).trim(); + if (StringUtil.hasText(part)) { + parts.add(part); } - if (current.length() > 0 && current.length() + text.length() + 2 > size) { - parts.add(current.toString().trim()); - current = new StringBuilder(); - } - if (current.length() > 0) { - current.append("\n\n"); - } - current.append(text); - } - if (current.length() > 0) { - parts.add(current.toString().trim()); - } - if (parts.isEmpty()) { - parts.add(content); } return parts; } + /** + * Applies the shared BGE-M3 context safety guard after strategy-specific splitting. + * + * @param chunks chunks produced by a semantic or length strategy + * @return chunks whose conservative token estimates stay within the hard limit + */ + private List enforceBgeM3HardLimit(List chunks) { + List result = new ArrayList(); + for (RagChunk chunk : chunks) { + String content = chunk.getContent(); + if (BgeM3ChunkSafety.isWithinHardLimit(content)) { + result.add(chunk); + continue; + } + List parts = + BgeM3ChunkSafety.splitToHardLimit(content); + for (int i = 0; i < parts.size(); i++) { + BgeM3ChunkSafety.ChunkRange range = parts.get(i); + result.add(copyHardSplitChunk( + chunk, + content, + new TextRange(range.getStart(), range.getEnd()), + i + 1, + parts.size())); + } + } + return result; + } + + /** + * Copies a strategy chunk while replacing only content and hard-split metadata. + * + * @param source original strategy chunk + * @param sourceContent original chunk content + * @param range selected content range + * @param partNo forced part number + * @param partTotal forced part count + * @return copied hard-split chunk + */ + private RagChunk copyHardSplitChunk(RagChunk source, + String sourceContent, + TextRange range, + int partNo, + int partTotal) { + RagChunk copy = new RagChunk(); + copy.setChunkType(source.getChunkType()); + copy.setSourceLabel(source.getSourceLabel()); + copy.setHeadingPath(new ArrayList(source.getHeadingPath())); + copy.setContent(sourceContent.substring(range.start, range.end)); + copy.setQuestion(source.getQuestion()); + copy.setAnswer(RagChunkTypes.QA_PAIR.equals(source.getChunkType()) + ? copy.getContent() + : source.getAnswer()); + copy.setPartNo(Integer.valueOf(partNo)); + copy.setPartTotal(Integer.valueOf(partTotal)); + copy.setWarnings(new ArrayList(source.getWarnings())); + copy.setOptions(new LinkedHashMap(source.getOptions())); + copy.getOptions().put("hardSplit", Boolean.TRUE); + copy.getOptions().put("hardSplitTokenLimit", + Integer.valueOf(RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT)); + adjustSingleSourceRange(copy, sourceContent.length(), range); + return copy; + } + + /** + * Narrows an exact single source range to the forced subrange. + * + * @param chunk copied hard-split chunk + * @param sourceLength original chunk content length + * @param relativeRange forced range relative to the original chunk + */ + @SuppressWarnings("unchecked") + private void adjustSingleSourceRange(RagChunk chunk, int sourceLength, TextRange relativeRange) { + Object rawRanges = chunk.getOptions().get(RagMetadataKeys.SOURCE_RANGES); + if (!(rawRanges instanceof List) || ((List) rawRanges).size() != 1) { + return; + } + Object rawRange = ((List) rawRanges).get(0); + if (!(rawRange instanceof Map)) { + return; + } + Map sourceRange = (Map) rawRange; + Object rawStart = sourceRange.get("start"); + Object rawEnd = sourceRange.get("end"); + if (!(rawStart instanceof Number) || !(rawEnd instanceof Number)) { + return; + } + int sourceStart = ((Number) rawStart).intValue(); + int sourceEnd = ((Number) rawEnd).intValue(); + if (sourceEnd - sourceStart != sourceLength) { + return; + } + chunk.getOptions().put(RagMetadataKeys.SOURCE_RANGES, + toSourceRangeMaps(Collections.singletonList( + new TextRange(sourceStart + relativeRange.start, sourceStart + relativeRange.end)))); + } + private List postProcess(List chunks) { List result = new ArrayList(); Set dedup = new HashSet(); @@ -318,7 +452,8 @@ public class RagSplitStrategyRegistry { } chunk.setChunkId("chunk-" + index); chunk.setCharCount(Integer.valueOf(content.length())); - chunk.setTokenEstimate(Integer.valueOf(Math.max(1, content.length() / 4))); + chunk.setTokenEstimate(Integer.valueOf( + Math.max(1, BgeM3ChunkSafety.estimateContentTokens(content)))); result.add(chunk); index++; } @@ -460,7 +595,14 @@ public class RagSplitStrategyRegistry { private int safeOverlap(StrategyConfig strategyConfig) { Integer overlapSize = strategyConfig.getOverlapSize(); - return overlapSize == null || overlapSize.intValue() < 0 ? RagDefaults.OVERLAP_SIZE : overlapSize.intValue(); + int overlap = overlapSize == null || overlapSize.intValue() < 0 + ? RagDefaults.OVERLAP_SIZE + : overlapSize.intValue(); + int chunkSize = safeChunkSize(strategyConfig); + if (overlap >= chunkSize) { + throw new IllegalArgumentException("overlapSize must be smaller than chunkSize"); + } + return overlap; } private String joinAndTrim(List lines) { diff --git a/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/model/StrategyConfig.java b/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/model/StrategyConfig.java index b4d4da2..07971f0 100644 --- a/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/model/StrategyConfig.java +++ b/easy-agents-rag/easy-agents-rag-ingestion/src/main/java/com/easyagents/rag/ingestion/model/StrategyConfig.java @@ -11,6 +11,10 @@ public class StrategyConfig implements Serializable { private Integer chunkSize = RagDefaults.CHUNK_SIZE; private Integer overlapSize = RagDefaults.OVERLAP_SIZE; private String regex; + /** + * Whether a custom-regex match is retained at the beginning of the next chunk. + */ + private Boolean retainRegexMatch; private Integer rowsPerChunk = RagDefaults.ROWS_PER_CHUNK; private Integer mdSplitterLevel = RagDefaults.MD_SPLITTER_LEVEL; @@ -24,6 +28,7 @@ public class StrategyConfig implements Serializable { copy.setChunkSize(this.chunkSize); copy.setOverlapSize(this.overlapSize); copy.setRegex(this.regex); + copy.setRetainRegexMatch(this.retainRegexMatch); copy.setRowsPerChunk(this.rowsPerChunk); copy.setMdSplitterLevel(this.mdSplitterLevel); return copy; @@ -61,6 +66,25 @@ public class StrategyConfig implements Serializable { this.regex = regex; } + /** + * Returns whether custom-regex matches are retained in the next chunk. + * + * @return {@code true} to retain each match at the beginning of the next chunk; + * otherwise {@code false} or {@code null} + */ + public Boolean getRetainRegexMatch() { + return retainRegexMatch; + } + + /** + * Sets whether custom-regex matches are retained in the next chunk. + * + * @param retainRegexMatch {@code true} to retain each match; {@code false} to discard it + */ + public void setRetainRegexMatch(Boolean retainRegexMatch) { + this.retainRegexMatch = retainRegexMatch; + } + public Integer getRowsPerChunk() { return rowsPerChunk; } diff --git a/easy-agents-rag/easy-agents-rag-ingestion/src/test/java/com/easyagents/rag/ingestion/RagIngestionPipelineTest.java b/easy-agents-rag/easy-agents-rag-ingestion/src/test/java/com/easyagents/rag/ingestion/RagIngestionPipelineTest.java index d1d94f3..669b024 100644 --- a/easy-agents-rag/easy-agents-rag-ingestion/src/test/java/com/easyagents/rag/ingestion/RagIngestionPipelineTest.java +++ b/easy-agents-rag/easy-agents-rag-ingestion/src/test/java/com/easyagents/rag/ingestion/RagIngestionPipelineTest.java @@ -2,6 +2,7 @@ package com.easyagents.rag.ingestion; import com.easyagents.rag.core.RagChunk; import com.easyagents.rag.core.RagChunkTypes; +import com.easyagents.rag.core.RagDefaults; import com.easyagents.rag.core.RagStrategyCodes; import com.easyagents.rag.ingestion.analysis.DocumentStructureAnalyzer; import com.easyagents.rag.ingestion.chunk.RagSplitStrategyRegistry; @@ -97,6 +98,157 @@ public class RagIngestionPipelineTest { assertHasValidSourceRanges(analysis, chunks.get(0)); } + @Test + public void shouldEnforceBgeM3HardLimitForAutoQaTxt() { + StringBuilder qa = new StringBuilder("问:自动导入为什么失败?\n答:"); + int i = 0; + while (qa.length() < 77946) { + qa.append((char) ('一' + i % 20)); + if (i > 0 && i % 700 == 0) { + qa.append('。'); + } + i++; + } + qa.setLength(77946); + AnalysisResult analysis = recommender.recommend(analyzer.analyze(qa.toString(), "txt")); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.AUTO); + + List chunks = registry.split(analysis, config); + + Assert.assertEquals(RagStrategyCodes.QA_PAIR, analysis.getRecommendedStrategyCode()); + Assert.assertEquals(77946, analysis.getNormalizedContent().length()); + Assert.assertTrue(chunks.size() > 1); + assertWithinBgeM3HardLimit(chunks); + Assert.assertTrue(chunks.stream().allMatch( + chunk -> chunk.getTokenEstimate().intValue() <= RagDefaults.CHUNK_SIZE)); + Assert.assertTrue(chunks.stream().allMatch( + chunk -> chunk.getAnswer() != null + && chunk.getAnswer().length() < analysis.getNormalizedContent().length())); + assertHasValidSourceRanges(analysis, chunks.get(0)); + } + + @Test + public void shouldEnforceBgeM3HardLimitForCustomRegexChunk() { + StringBuilder content = new StringBuilder(); + for (int i = 0; i < 20000; i++) { + content.append((char) ('甲' + i % 16)); + if (i > 0 && i % 997 == 0) { + content.append(';'); + } + } + AnalysisResult analysis = recommender.recommend(analyzer.analyze(content.toString(), "txt")); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.CUSTOM_REGEX); + config.setRegex("\\|NEVER_MATCH\\|"); + + List chunks = registry.split(analysis, config); + + Assert.assertTrue(chunks.size() > 1); + assertWithinBgeM3HardLimit(chunks); + assertHasValidSourceRanges(analysis, chunks.get(0)); + } + + /** + * Verifies that custom-regex matches remain discarded when the new option is omitted. + */ + @Test + public void shouldDiscardCustomRegexMatchByDefault() { + AnalysisResult analysis = new AnalysisResult(); + analysis.setNormalizedContent( + "第一个问题及其答案内容。第二个问题及其答案内容。"); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.CUSTOM_REGEX); + config.setRegex(""); + + List chunks = registry.split(analysis, config); + + Assert.assertEquals(2, chunks.size()); + Assert.assertEquals("第一个问题及其答案内容。", chunks.get(0).getContent()); + Assert.assertEquals("第二个问题及其答案内容。", chunks.get(1).getContent()); + } + + /** + * Verifies that an enabled option retains each match at the beginning of the next chunk. + */ + @Test + public void shouldRetainCustomRegexMatchInNextChunk() { + AnalysisResult analysis = new AnalysisResult(); + analysis.setNormalizedContent( + "第一个问题及其答案内容。第二个问题及其答案内容。"); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.CUSTOM_REGEX); + config.setRegex(""); + config.setRetainRegexMatch(Boolean.TRUE); + + List chunks = registry.split(analysis, config); + + Assert.assertEquals(2, chunks.size()); + Assert.assertEquals( + "第一个问题及其答案内容。", + chunks.get(0).getContent()); + Assert.assertEquals( + "第二个问题及其答案内容。", + chunks.get(1).getContent()); + assertHasValidSourceRanges(analysis, chunks.get(1)); + } + + @Test + public void shouldUseChineseAwareTokenEstimate() { + String content = "知识库自动导入向量化失败,需要检查分块大小和模型上下文窗口。"; + AnalysisResult analysis = recommender.recommend(analyzer.analyze(content, "txt")); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH); + + List chunks = registry.split(analysis, config); + + Assert.assertEquals(Integer.valueOf(content.codePointCount(0, content.length())), + chunks.get(0).getTokenEstimate()); + } + + @Test + public void shouldRejectOverlapEqualToChunkSize() { + AnalysisResult analysis = recommender.recommend( + analyzer.analyze("知识库分块参数需要保证游标持续向前推进。", "txt")); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH); + config.setChunkSize(128); + config.setOverlapSize(128); + + try { + registry.split(analysis, config); + Assert.fail("overlapSize 等于 chunkSize 时应拒绝分块"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("overlapSize")); + } + } + + @Test + public void shouldRejectOverlapGreaterThanChunkSize() { + AnalysisResult analysis = recommender.recommend( + analyzer.analyze("知识库分块参数需要保证游标持续向前推进。", "txt")); + StrategyConfig config = StrategyConfig.defaults(); + config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH); + config.setChunkSize(128); + config.setOverlapSize(512); + + try { + registry.split(analysis, config); + Assert.fail("overlapSize 大于 chunkSize 时应拒绝分块"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("overlapSize")); + } + } + + private void assertWithinBgeM3HardLimit(List chunks) { + for (RagChunk chunk : chunks) { + Assert.assertNotNull(chunk.getTokenEstimate()); + Assert.assertTrue( + "chunk token estimate exceeds BGE-M3 hard limit: " + chunk.getTokenEstimate(), + chunk.getTokenEstimate().intValue() <= RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT); + } + } + @SuppressWarnings("unchecked") private void assertHasValidSourceRanges(AnalysisResult analysis, RagChunk chunk) { Object rawRanges = chunk.getOptions().get("sourceRanges"); diff --git a/easy-agents-search-engine/easy-agents-search-engine-es/src/main/java/com/easyagents/engine/es/ElasticSearcher.java b/easy-agents-search-engine/easy-agents-search-engine-es/src/main/java/com/easyagents/engine/es/ElasticSearcher.java index 7f7f07a..f3a8190 100644 --- a/easy-agents-search-engine/easy-agents-search-engine-es/src/main/java/com/easyagents/engine/es/ElasticSearcher.java +++ b/easy-agents-search-engine/easy-agents-search-engine-es/src/main/java/com/easyagents/engine/es/ElasticSearcher.java @@ -2,8 +2,6 @@ package com.easyagents.engine.es; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch.core.*; -import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; -import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; import co.elastic.clients.elasticsearch.core.search.SourceConfig; import co.elastic.clients.json.JsonData; import co.elastic.clients.json.jackson.JacksonJsonpMapper; @@ -25,19 +23,46 @@ import org.slf4j.LoggerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import java.io.IOException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.X509Certificate; import java.util.*; -public class ElasticSearcher implements DocumentSearcher { +/** + * 基于 Elasticsearch 的关键词搜索器。 + * + *

底层客户端线程安全并与搜索器实例共享生命周期,避免逐文档重复创建网络连接。

+ */ +public class ElasticSearcher implements DocumentSearcher, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ElasticSearcher.class); + static final int MAX_BULK_OPERATIONS = 200; + static final long MAX_BULK_ESTIMATED_BYTES = 5L * 1024L * 1024L; + private static final long BULK_OPERATION_METADATA_BYTES = 128L; private final ESConfig esConfig; + private final JacksonJsonpMapper jsonpMapper = new JacksonJsonpMapper(); + private final ElasticsearchTransport transport; + private final ElasticsearchClient client; + /** + * 创建 Elasticsearch 搜索器及其共享客户端。 + * + * @param esConfig Elasticsearch 配置 + * @throws IllegalStateException 客户端初始化失败时抛出 + */ public ElasticSearcher(ESConfig esConfig) { - this.esConfig = esConfig; + this.esConfig = Objects.requireNonNull(esConfig, "ESConfig 不能为 null"); + RestClient openedRestClient = null; + try { + openedRestClient = buildRestClient(); + this.transport = new RestClientTransport(openedRestClient, jsonpMapper); + this.client = new ElasticsearchClient(transport); + } catch (Exception e) { + closeQuietly(openedRestClient); + throw new IllegalStateException("初始化 Elasticsearch 客户端失败", e); + } } // 忽略 SSL 的 client 构建逻辑 @@ -76,52 +101,128 @@ public class ElasticSearcher implements DocumentSearcher { /** - * 添加文档到Elasticsearch + * 添加文档到 Elasticsearch。 + * + * @param document 待写入文档 + * @return 写入成功时返回 {@code true} */ @Override public boolean addDocument(Document document) { - if (document == null || document.getContent() == null) { + return addDocuments(Collections.singletonList(document)); + } + + /** + * 使用有界 Bulk 请求批量写入文档。 + * + * @param documents 待写入文档 + * @return 全部文档写入成功时返回 {@code true} + */ + @Override + public boolean addDocuments(List documents) { + if (!areValidDocuments(documents)) { return false; } - - RestClient restClient = null; - ElasticsearchTransport transport = null; try { - restClient = buildRestClient(); - transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); - ElasticsearchClient client = new ElasticsearchClient(transport); - - Map source = buildSource(document); - String documentId = document.getId().toString(); - IndexOperation indexOp = IndexOperation.of(i -> i - .index(esConfig.getIndexName()) - .id(documentId) - .document(JsonData.of(source)) - ); - - BulkOperation bulkOp = BulkOperation.of(b -> b.index(indexOp)); - BulkRequest request = BulkRequest.of(b -> b.operations(Collections.singletonList(bulkOp))); - BulkResponse response = client.bulk(request); - return !response.errors(); - + List> batches = partitionDocuments(documents); + for (int batchIndex = 0; batchIndex < batches.size(); batchIndex++) { + List batch = batches.get(batchIndex); + BulkResponse response = client.bulk(buildBulkRequest(batch)); + if (!isBulkSuccessful(response, "写入", batchIndex, batch.size())) { + return false; + } + } + return true; } catch (Exception e) { - LOG.error(e.getMessage(), e); + LOG.error("Elasticsearch 批量写入异常: count={}", documents.size(), e); return false; - } finally { - closeResources(transport, restClient); } } + /** + * 按操作数量和序列化后的估算字节数拆分 Bulk 批次。 + * + * @param documents 已完成校验的文档 + * @return 有界文档批次 + * @throws IOException 文档序列化失败时抛出 + * @throws IllegalArgumentException 单个文档超过批次字节限制时抛出 + */ + List> partitionDocuments(List documents) throws IOException { + List> batches = new ArrayList<>(); + List currentBatch = new ArrayList<>(Math.min(documents.size(), MAX_BULK_OPERATIONS)); + long currentBatchBytes = 0L; + for (Document document : documents) { + long documentBytes = estimateBulkOperationBytes(document); + if (documentBytes > MAX_BULK_ESTIMATED_BYTES) { + throw new IllegalArgumentException( + "单个 Elasticsearch 索引文档超过 Bulk 字节限制: id=" + document.getId()); + } + if (!currentBatch.isEmpty() + && (currentBatch.size() >= MAX_BULK_OPERATIONS + || currentBatchBytes + documentBytes > MAX_BULK_ESTIMATED_BYTES)) { + batches.add(currentBatch); + currentBatch = new ArrayList<>(Math.min(documents.size(), MAX_BULK_OPERATIONS)); + currentBatchBytes = 0L; + } + currentBatch.add(document); + currentBatchBytes += documentBytes; + } + if (!currentBatch.isEmpty()) { + batches.add(currentBatch); + } + return batches; + } + + private long estimateBulkOperationBytes(Document document) throws IOException { + long sourceBytes = jsonpMapper.objectMapper().writeValueAsBytes(buildSource(document)).length; + return sourceBytes + + utf8Length(esConfig.getIndexName()) + + utf8Length(document.getId().toString()) + + BULK_OPERATION_METADATA_BYTES; + } + + private int utf8Length(String value) { + return value == null ? 0 : value.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + } + + /** + * 构建包含全部文档的 Bulk 请求。 + * + * @param documents 已完成校验的文档 + * @return Bulk 请求 + */ + BulkRequest buildBulkRequest(List documents) { + BulkRequest.Builder builder = new BulkRequest.Builder(); + for (Document document : documents) { + builder.operations(operation -> operation.index(index -> index + .index(esConfig.getIndexName()) + .id(document.getId().toString()) + .document(JsonData.of(buildSource(document))) + )); + } + return builder.build(); + } + + private boolean areValidDocuments(List documents) { + if (documents == null || documents.isEmpty()) { + return false; + } + for (Document document : documents) { + if (document == null || document.getId() == null || document.getContent() == null) { + return false; + } + } + return true; + } + + /** + * 按请求条件搜索文档。 + * + * @param request 搜索请求 + * @return 命中文档 + */ @Override public List searchDocuments(KeywordSearchRequest request) { - RestClient restClient = null; - ElasticsearchTransport transport = null; - try { - restClient = buildRestClient(); - transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); - ElasticsearchClient client = new ElasticsearchClient(transport); - SearchResponse response = client.search(buildSearchRequest(request), Map.class); List results = new ArrayList<>(); response.hits().hits().forEach(hit -> { @@ -137,54 +238,114 @@ public class ElasticSearcher implements DocumentSearcher { } catch (Exception e) { LOG.error(e.getMessage(), e); return Collections.emptyList(); - } finally { - closeResources(transport, restClient); } } + /** + * 删除指定文档。 + * + * @param id 文档 ID + * @return 删除成功时返回 {@code true} + */ @Override public boolean deleteDocument(Object id) { - if (id == null) { + return deleteDocuments(Collections.singletonList(id)); + } + + /** + * 使用有界 Bulk 请求批量删除文档。 + * + *

删除不存在的文档按幂等成功处理;仅 Bulk 条目包含实际错误时返回失败。

+ * + * @param ids 文档 ID 集合 + * @return 全部删除操作成功时返回 {@code true} + */ + @Override + public boolean deleteDocuments(Collection ids) { + if (ids == null || ids.isEmpty()) { return false; } - - RestClient restClient = null; - ElasticsearchTransport transport = null; + List documentIds = new ArrayList<>(ids.size()); + for (Object id : ids) { + if (id == null) { + return false; + } + documentIds.add(id.toString()); + } try { - restClient = buildRestClient(); - transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); - ElasticsearchClient client = new ElasticsearchClient(transport); - - DeleteRequest request = DeleteRequest.of(d -> d - .index(esConfig.getIndexName()) - .id(id.toString()) - ); - - DeleteResponse response = client.delete(request); - return response.result() == co.elastic.clients.elasticsearch._types.Result.Deleted; - + for (int start = 0, batchIndex = 0; + start < documentIds.size(); + start += MAX_BULK_OPERATIONS, batchIndex++) { + int end = Math.min(start + MAX_BULK_OPERATIONS, documentIds.size()); + List batch = documentIds.subList(start, end); + BulkResponse response = client.bulk(buildDeleteBulkRequest(batch)); + if (!isBulkSuccessful(response, "删除", batchIndex, batch.size())) { + return false; + } + } + return true; } catch (Exception e) { - LOG.error("Error deleting document with id: " + id, e); + LOG.error("Elasticsearch 批量删除异常: count={}", documentIds.size(), e); return false; - } finally { - closeResources(transport, restClient); } } + /** + * 构建批量删除请求。 + * + * @param ids 文档 ID + * @return Bulk 删除请求 + */ + BulkRequest buildDeleteBulkRequest(List ids) { + BulkRequest.Builder builder = new BulkRequest.Builder(); + for (String id : ids) { + builder.operations(operation -> operation.delete(delete -> delete + .index(esConfig.getIndexName()) + .id(id) + )); + } + return builder.build(); + } + + private boolean isBulkSuccessful(BulkResponse response, + String operation, + int batchIndex, + int batchSize) { + if (response != null && !response.errors()) { + return true; + } + if (response == null) { + LOG.error("Elasticsearch 批量{}未返回结果: batchIndex={}, batchSize={}", + operation, batchIndex, batchSize); + return false; + } + response.items().stream() + .filter(item -> item.error() != null) + .forEach(item -> LOG.error( + "Elasticsearch 批量{}失败: batchIndex={}, index={}, id={}, status={}, reason={}", + operation, + batchIndex, + item.index(), + item.id(), + item.status(), + item.error().reason() + )); + return false; + } + + /** + * 更新指定文档。 + * + * @param document 待更新文档 + * @return 更新成功时返回 {@code true} + */ @Override public boolean updateDocument(Document document) { if (document == null || document.getId() == null) { return false; } - RestClient restClient = null; - ElasticsearchTransport transport = null; - try { - restClient = buildRestClient(); - transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); - ElasticsearchClient client = new ElasticsearchClient(transport); - UpdateRequest, Map> request = UpdateRequest.of(u -> u .index(esConfig.getIndexName()) .id(document.getId().toString()) @@ -199,20 +360,17 @@ public class ElasticSearcher implements DocumentSearcher { } catch (Exception e) { LOG.error("Error updating document with id: " + document.getId(), e); return false; - } finally { - closeResources(transport, restClient); } } - - private void closeResources(AutoCloseable... closeables) { - for (AutoCloseable closeable : closeables) { - try { - if (closeable != null) - closeable.close(); - } catch (Exception e) { - LOG.error("Error closing resource", e); - } + private static void closeQuietly(AutoCloseable closeable) { + if (closeable == null) { + return; + } + try { + closeable.close(); + } catch (Exception ignored) { + // 初始化失败时仅执行尽力清理,原始异常由构造方法继续抛出。 } } @@ -284,19 +442,27 @@ public class ElasticSearcher implements DocumentSearcher { ); } + /** + * 检查 Elasticsearch 服务是否可用。 + * + * @return 服务可访问时返回 {@code true} + */ public boolean checkAvailable() { - RestClient restClient = null; - ElasticsearchTransport transport = null; try { - restClient = buildRestClient(); - transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); - ElasticsearchClient client = new ElasticsearchClient(transport); return client.info() != null; } catch (Exception e) { LOG.error("Elasticsearch availability check failed", e); return false; - } finally { - closeResources(transport, restClient); } } + + /** + * 关闭共享传输层及其底层 RestClient。 + * + * @throws IOException 关闭客户端资源失败时抛出 + */ + @Override + public void close() throws IOException { + transport.close(); + } } diff --git a/easy-agents-search-engine/easy-agents-search-engine-es/src/test/java/com/easyagents/engine/es/ElasticSearcherQueryBuilderTest.java b/easy-agents-search-engine/easy-agents-search-engine-es/src/test/java/com/easyagents/engine/es/ElasticSearcherQueryBuilderTest.java index 31b929d..a3c1ecc 100644 --- a/easy-agents-search-engine/easy-agents-search-engine-es/src/test/java/com/easyagents/engine/es/ElasticSearcherQueryBuilderTest.java +++ b/easy-agents-search-engine/easy-agents-search-engine-es/src/test/java/com/easyagents/engine/es/ElasticSearcherQueryBuilderTest.java @@ -1,48 +1,185 @@ package com.easyagents.engine.es; import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.BulkRequest; import com.easyagents.core.document.Document; import com.easyagents.search.engine.service.KeywordSearchMetadataKeys; import com.easyagents.search.engine.service.KeywordSearchRequest; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +/** + * {@link ElasticSearcher} 请求构建回归测试。 + */ public class ElasticSearcherQueryBuilderTest { + /** + * 验证搜索请求同时包含多字段检索和知识库过滤。 + * + * @throws Exception Elasticsearch 客户端关闭失败时抛出 + */ @Test - public void shouldBuildSearchRequestWithMultiMatchAndKnowledgeFilter() { - ElasticSearcher searcher = new ElasticSearcher(config()); - KeywordSearchRequest request = KeywordSearchRequest.of("客服", 5); - request.setKnowledgeId("100"); + public void shouldBuildSearchRequestWithMultiMatchAndKnowledgeFilter() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + KeywordSearchRequest request = KeywordSearchRequest.of("客服", 5); + request.setKnowledgeId("100"); - SearchRequest searchRequest = searcher.buildSearchRequest(request); + SearchRequest searchRequest = searcher.buildSearchRequest(request); - Assert.assertEquals(5, searchRequest.size().intValue()); - Assert.assertNotNull(searchRequest.query().bool()); - Assert.assertEquals(1, searchRequest.query().bool().must().size()); - Assert.assertNotNull(searchRequest.query().bool().must().get(0).multiMatch()); - Assert.assertEquals(2, searchRequest.query().bool().must().get(0).multiMatch().fields().size()); - Assert.assertEquals(1, searchRequest.query().bool().filter().size()); - Assert.assertEquals("knowledgeId", searchRequest.query().bool().filter().get(0).term().field()); + Assert.assertEquals(5, searchRequest.size().intValue()); + Assert.assertNotNull(searchRequest.query().bool()); + Assert.assertEquals(1, searchRequest.query().bool().must().size()); + Assert.assertNotNull(searchRequest.query().bool().must().get(0).multiMatch()); + Assert.assertEquals(2, searchRequest.query().bool().must().get(0).multiMatch().fields().size()); + Assert.assertEquals(1, searchRequest.query().bool().filter().size()); + Assert.assertEquals("knowledgeId", searchRequest.query().bool().filter().get(0).term().field()); + } } + /** + * 验证知识库 ID 会写入顶层字段。 + * + * @throws Exception Elasticsearch 客户端关闭失败时抛出 + */ @Test - public void shouldExtractKnowledgeIdToTopLevelSource() { - ElasticSearcher searcher = new ElasticSearcher(config()); + public void shouldExtractKnowledgeIdToTopLevelSource() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + Document document = new Document(); + document.setId("1"); + document.setTitle("title"); + document.setContent("content"); + document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100"); + + Map source = searcher.buildSource(document); + + Assert.assertEquals("100", source.get(KeywordSearchMetadataKeys.KNOWLEDGE_ID)); + Assert.assertTrue(source.get("metadataMap") instanceof Map); + } + } + + /** + * 验证多个文档会进入同一个 Bulk 请求并保留稳定文档 ID。 + * + * @throws Exception Elasticsearch 客户端关闭失败时抛出 + */ + @Test + public void shouldBuildSingleBulkRequestForMultipleDocuments() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + Document first = new Document(); + first.setId("first"); + first.setContent("first content"); + Document second = new Document(); + second.setId("second"); + second.setContent("second content"); + + BulkRequest request = searcher.buildBulkRequest(List.of(first, second)); + + Assert.assertEquals(2, request.operations().size()); + Assert.assertEquals("first", request.operations().get(0).index().id()); + Assert.assertEquals("second", request.operations().get(1).index().id()); + } + } + + /** + * 验证 Bulk 写入会按最大操作数拆分,避免单次请求无界增长。 + * + * @throws Exception Elasticsearch 客户端关闭或文档序列化失败时抛出 + */ + @Test + public void shouldPartitionBulkRequestsByOperationCount() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + List documents = new ArrayList<>(); + for (int index = 0; index <= ElasticSearcher.MAX_BULK_OPERATIONS; index++) { + documents.add(document("count-" + index, "content")); + } + + List> batches = searcher.partitionDocuments(documents); + + Assert.assertEquals(2, batches.size()); + Assert.assertEquals(ElasticSearcher.MAX_BULK_OPERATIONS, batches.get(0).size()); + Assert.assertEquals(1, batches.get(1).size()); + } + } + + /** + * 验证 Bulk 写入会按序列化字节数拆分。 + * + * @throws Exception Elasticsearch 客户端关闭或文档序列化失败时抛出 + */ + @Test + public void shouldPartitionBulkRequestsByEstimatedBytes() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + int contentLength = (int) (ElasticSearcher.MAX_BULK_ESTIMATED_BYTES / 2L); + Document first = document("bytes-first", "a".repeat(contentLength)); + Document second = document("bytes-second", "b".repeat(contentLength)); + + List> batches = searcher.partitionDocuments(List.of(first, second)); + + Assert.assertEquals(2, batches.size()); + Assert.assertEquals(1, batches.get(0).size()); + Assert.assertEquals(1, batches.get(1).size()); + } + } + + /** + * 验证单个超限文档会在发送请求前失败。 + * + * @throws Exception Elasticsearch 客户端关闭或文档序列化失败时抛出 + */ + @Test + public void shouldRejectSingleDocumentOverByteLimit() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + int contentLength = (int) ElasticSearcher.MAX_BULK_ESTIMATED_BYTES; + Document oversized = document("oversized", "a".repeat(contentLength)); + + try { + searcher.partitionDocuments(List.of(oversized)); + Assert.fail("单个超限文档应拒绝构建 Bulk 请求"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("oversized")); + } + } + } + + /** + * 验证批量删除请求包含全部稳定文档 ID。 + * + * @throws Exception Elasticsearch 客户端关闭失败时抛出 + */ + @Test + public void shouldBuildBulkDeleteRequest() throws Exception { + try (ElasticSearcher searcher = new ElasticSearcher(config())) { + BulkRequest request = searcher.buildDeleteBulkRequest(List.of("first", "second")); + + Assert.assertEquals(2, request.operations().size()); + Assert.assertEquals("first", request.operations().get(0).delete().id()); + Assert.assertEquals("second", request.operations().get(1).delete().id()); + } + } + + /** + * 创建测试文档。 + * + * @param id 文档 ID + * @param content 文档内容 + * @return 文档 + */ + private Document document(String id, String content) { Document document = new Document(); - document.setId("1"); - document.setTitle("title"); - document.setContent("content"); - document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100"); - - Map source = searcher.buildSource(document); - - Assert.assertEquals("100", source.get(KeywordSearchMetadataKeys.KNOWLEDGE_ID)); - Assert.assertTrue(source.get("metadataMap") instanceof Map); + document.setId(id); + document.setContent(content); + return document; } + /** + * 创建测试用 Elasticsearch 配置。 + * + * @return Elasticsearch 配置 + */ private ESConfig config() { ESConfig config = new ESConfig(); config.setHost("http://127.0.0.1:9200"); diff --git a/easy-agents-search-engine/easy-agents-search-engine-lucene/src/main/java/com/easyagents/search/engine/lucene/LuceneSearcher.java b/easy-agents-search-engine/easy-agents-search-engine-lucene/src/main/java/com/easyagents/search/engine/lucene/LuceneSearcher.java index 5edd49e..84c4a7c 100644 --- a/easy-agents-search-engine/easy-agents-search-engine-lucene/src/main/java/com/easyagents/search/engine/lucene/LuceneSearcher.java +++ b/easy-agents-search-engine/easy-agents-search-engine-lucene/src/main/java/com/easyagents/search/engine/lucene/LuceneSearcher.java @@ -29,7 +29,7 @@ import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; -import org.jetbrains.annotations.NotNull; +import org.apache.lucene.util.IOUtils; import org.lionsoul.jcseg.ISegment; import org.lionsoul.jcseg.analyzer.JcsegAnalyzer; import org.lionsoul.jcseg.dic.DictionaryFactory; @@ -40,17 +40,38 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.Set; -public class LuceneSearcher implements DocumentSearcher { +/** + * 基于 Lucene 的本地关键词搜索器。 + * + *

每个实例长期复用一个线程安全的 {@link IndexWriter},避免并发任务重复抢占 + * 同一索引目录的 {@code write.lock}。

+ */ +public class LuceneSearcher implements DocumentSearcher, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(LuceneSearcher.class); - private Directory directory; + private final Directory directory; + private final Analyzer analyzer; + private final IndexWriter indexWriter; + /** + * 创建 Lucene 搜索器并打开索引写入器。 + * + * @param config Lucene 配置 + * @throws IllegalStateException 索引目录或写入器初始化失败时抛出 + */ public LuceneSearcher(LuceneConfig config) { Objects.requireNonNull(config, "LuceneConfig 不能为 null"); + Directory openedDirectory = null; + Analyzer openedAnalyzer = null; + IndexWriter openedWriter = null; try { String indexDirPath = config.getIndexDirPath(); // 索引目录路径 File indexDir = new File(indexDirPath); @@ -58,89 +79,123 @@ public class LuceneSearcher implements DocumentSearcher { throw new IllegalStateException("can not mkdirs for path: " + indexDirPath); } - this.directory = FSDirectory.open(indexDir.toPath()); - } catch (IOException e) { + openedDirectory = FSDirectory.open(indexDir.toPath()); + openedAnalyzer = createAnalyzer(); + openedWriter = new IndexWriter(openedDirectory, new IndexWriterConfig(openedAnalyzer)); + } catch (IOException | RuntimeException e) { + IOUtils.closeWhileHandlingException(openedWriter, openedAnalyzer, openedDirectory); LOG.error("初始化 Lucene 索引失败", e); - throw new RuntimeException(e); + throw new IllegalStateException("初始化 Lucene 索引失败", e); } + this.directory = openedDirectory; + this.analyzer = openedAnalyzer; + this.indexWriter = openedWriter; } + /** + * 以文档 ID 为键写入或覆盖单个文档。 + * + * @param document 待写入文档 + * @return 写入成功时返回 {@code true} + */ @Override public boolean addDocument(Document document) { - if (document == null || document.getContent() == null) return false; + return addDocuments(Collections.singletonList(document)); + } - IndexWriter indexWriter = null; - try { - indexWriter = createIndexWriter(); - - org.apache.lucene.document.Document luceneDoc = new org.apache.lucene.document.Document(); - luceneDoc.add(new StringField("id", document.getId().toString(), Field.Store.YES)); - luceneDoc.add(new TextField("content", document.getContent(), Field.Store.YES)); - - if (document.getTitle() != null) { - luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES)); + /** + * 批量写入或覆盖文档,并在批次结束后统一提交。 + * + * @param documents 待写入文档 + * @return 全部文档写入成功时返回 {@code true} + */ + @Override + public boolean addDocuments(List documents) { + if (documents == null || documents.isEmpty()) { + return false; + } + List luceneDocuments = new ArrayList<>(documents.size()); + for (Document document : documents) { + if (document == null || document.getId() == null || document.getContent() == null) { + return false; + } + luceneDocuments.add(toLuceneDocument(document)); + } + try { + for (org.apache.lucene.document.Document luceneDocument : luceneDocuments) { + String documentId = luceneDocument.get("id"); + // updateDocument 以 ID 执行 upsert,确保失败重试不会生成重复索引。 + indexWriter.updateDocument(new Term("id", documentId), luceneDocument); } - appendKnowledgeId(document, luceneDoc); - - indexWriter.addDocument(luceneDoc); indexWriter.commit(); return true; } catch (Exception e) { - LOG.error("添加文档失败", e); + LOG.error("批量添加文档失败: count={}", documents.size(), e); return false; - } finally { - close(indexWriter); } } - + /** + * 删除指定文档。 + * + * @param id 文档 ID + * @return 删除成功时返回 {@code true} + */ @Override public boolean deleteDocument(Object id) { - if (id == null) return false; + return deleteDocuments(Collections.singletonList(id)); + } - IndexWriter indexWriter = null; + /** + * 批量删除指定文档,并在批次结束后统一提交。 + * + * @param ids 文档 ID 集合 + * @return 全部文档删除成功时返回 {@code true} + */ + @Override + public boolean deleteDocuments(Collection ids) { + if (ids == null || ids.isEmpty()) { + return false; + } + Set uniqueIds = new LinkedHashSet<>(); + for (Object id : ids) { + if (id == null) { + return false; + } + uniqueIds.add(id.toString()); + } try { - indexWriter = createIndexWriter(); - Term term = new Term("id", id.toString()); - indexWriter.deleteDocuments(term); + Term[] terms = new Term[uniqueIds.size()]; + int index = 0; + for (String id : uniqueIds) { + terms[index++] = new Term("id", id); + } + indexWriter.deleteDocuments(terms); indexWriter.commit(); return true; } catch (IOException e) { - LOG.error("删除文档失败", e); + LOG.error("批量删除文档失败: count={}", uniqueIds.size(), e); return false; - } finally { - close(indexWriter); } } + /** + * 以文档 ID 为键更新文档。 + * + * @param document 待更新文档 + * @return 更新成功时返回 {@code true} + */ @Override public boolean updateDocument(Document document) { - if (document == null || document.getId() == null) return false; - - IndexWriter indexWriter = null; - try { - indexWriter = createIndexWriter(); - Term term = new Term("id", document.getId().toString()); - - org.apache.lucene.document.Document luceneDoc = new org.apache.lucene.document.Document(); - luceneDoc.add(new StringField("id", document.getId().toString(), Field.Store.YES)); - luceneDoc.add(new TextField("content", document.getContent(), Field.Store.YES)); - - if (document.getTitle() != null) { - luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES)); - } - appendKnowledgeId(document, luceneDoc); - indexWriter.updateDocument(term, luceneDoc); - indexWriter.commit(); - return true; - } catch (IOException e) { - LOG.error("更新文档失败", e); - return false; - } finally { - close(indexWriter); - } + return addDocument(document); } + /** + * 按请求条件搜索文档。 + * + * @param request 搜索请求 + * @return 命中文档 + */ @Override public List searchDocuments(KeywordSearchRequest request) { List results = new ArrayList<>(); @@ -171,7 +226,6 @@ public class LuceneSearcher implements DocumentSearcher { Query buildQuery(KeywordSearchRequest request) { try { - Analyzer analyzer = createAnalyzer(); String keyword = request == null ? null : request.getKeyword(); QueryParser titleQueryParser = new QueryParser("title", analyzer); @@ -195,20 +249,22 @@ public class LuceneSearcher implements DocumentSearcher { return null; } - - @NotNull - private IndexWriter createIndexWriter() throws IOException { - Analyzer analyzer = createAnalyzer(); - IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); - return new IndexWriter(directory, indexWriterConfig); - } - - private static Analyzer createAnalyzer() { SegmenterConfig config = new SegmenterConfig(true); return new JcsegAnalyzer(ISegment.Type.NLP, config, DictionaryFactory.createSingletonDictionary(config)); } + private org.apache.lucene.document.Document toLuceneDocument(Document document) { + org.apache.lucene.document.Document luceneDoc = new org.apache.lucene.document.Document(); + luceneDoc.add(new StringField("id", document.getId().toString(), Field.Store.YES)); + luceneDoc.add(new TextField("content", document.getContent(), Field.Store.YES)); + if (document.getTitle() != null) { + luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES)); + } + appendKnowledgeId(document, luceneDoc); + return luceneDoc; + } + private void appendKnowledgeId(Document document, org.apache.lucene.document.Document luceneDoc) { if (document == null || document.getMetadataMap() == null) { return; @@ -219,13 +275,13 @@ public class LuceneSearcher implements DocumentSearcher { } } - public void close(IndexWriter indexWriter) { - try { - if (indexWriter != null) { - indexWriter.close(); - } - } catch (IOException e) { - LOG.error("关闭 Lucene 失败", e); - } + /** + * 关闭写入器、分词器和索引目录。 + * + * @throws IOException 任一 Lucene 资源关闭失败时抛出 + */ + @Override + public void close() throws IOException { + IOUtils.close(indexWriter, analyzer, directory); } } diff --git a/easy-agents-search-engine/easy-agents-search-engine-lucene/src/test/java/com/easyagents/search/engine/lucene/LuceneSearcherTest.java b/easy-agents-search-engine/easy-agents-search-engine-lucene/src/test/java/com/easyagents/search/engine/lucene/LuceneSearcherTest.java index 4f21723..e7804dc 100644 --- a/easy-agents-search-engine/easy-agents-search-engine-lucene/src/test/java/com/easyagents/search/engine/lucene/LuceneSearcherTest.java +++ b/easy-agents-search-engine/easy-agents-search-engine-lucene/src/test/java/com/easyagents/search/engine/lucene/LuceneSearcherTest.java @@ -8,38 +8,145 @@ import org.junit.Test; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +/** + * {@link LuceneSearcher} 回归测试。 + */ public class LuceneSearcherTest { + /** + * 验证知识库过滤同时覆盖标题和正文。 + * + * @throws Exception 临时目录或 Lucene 资源操作失败时抛出 + */ @Test public void shouldFilterByKnowledgeIdAndSearchTitleAndContent() throws Exception { Path tempDir = Files.createTempDirectory("lucene-searcher-test"); LuceneConfig config = new LuceneConfig(); config.setIndexDirPath(tempDir.toString()); - LuceneSearcher searcher = new LuceneSearcher(config); + try (LuceneSearcher searcher = new LuceneSearcher(config)) { + Document first = new Document(); + first.setId("1"); + first.setTitle("客服标题"); + first.setContent("这里没有关键字"); + first.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100"); - Document first = new Document(); - first.setId("1"); - first.setTitle("客服标题"); - first.setContent("这里没有关键字"); - first.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100"); + Document second = new Document(); + second.setId("2"); + second.setTitle("别的知识库"); + second.setContent("客服内容"); + second.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "200"); - Document second = new Document(); - second.setId("2"); - second.setTitle("别的知识库"); - second.setContent("客服内容"); - second.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "200"); + Assert.assertTrue(searcher.addDocument(first)); + Assert.assertTrue(searcher.addDocument(second)); - Assert.assertTrue(searcher.addDocument(first)); - Assert.assertTrue(searcher.addDocument(second)); + KeywordSearchRequest request = KeywordSearchRequest.of("客服", 10); + request.setKnowledgeId("100"); + List results = searcher.searchDocuments(request); - KeywordSearchRequest request = KeywordSearchRequest.of("客服", 10); - request.setKnowledgeId("100"); - List results = searcher.searchDocuments(request); + Assert.assertEquals(1, results.size()); + Assert.assertEquals("1", String.valueOf(results.get(0).getId())); + Assert.assertEquals("100", String.valueOf( + results.get(0).getMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID))); + } + } - Assert.assertEquals(1, results.size()); - Assert.assertEquals("1", String.valueOf(results.get(0).getId())); - Assert.assertEquals("100", String.valueOf(results.get(0).getMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID))); + /** + * 验证多个导入线程可共享同一个 IndexWriter 完成批量写入。 + * + * @throws Exception 并发执行或 Lucene 资源操作失败时抛出 + */ + @Test + public void shouldWriteConcurrentBatchesWithoutLockConflict() throws Exception { + Path tempDir = Files.createTempDirectory("lucene-searcher-concurrent-test"); + LuceneConfig config = new LuceneConfig(); + config.setIndexDirPath(tempDir.toString()); + int workerCount = 4; + int documentsPerWorker = 20; + ExecutorService executor = Executors.newFixedThreadPool(workerCount); + try (LuceneSearcher searcher = new LuceneSearcher(config)) { + CountDownLatch startSignal = new CountDownLatch(1); + List> futures = new ArrayList<>(); + for (int worker = 0; worker < workerCount; worker++) { + int currentWorker = worker; + futures.add(executor.submit(() -> { + startSignal.await(); + List documents = new ArrayList<>(); + for (int offset = 0; offset < documentsPerWorker; offset++) { + Document document = new Document(); + document.setId(currentWorker + "-" + offset); + document.setContent("concurrent indexing content"); + document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "concurrent-kb"); + documents.add(document); + } + return searcher.addDocuments(documents); + })); + } + startSignal.countDown(); + for (Future future : futures) { + Assert.assertTrue(future.get()); + } + + int expectedCount = workerCount * documentsPerWorker; + KeywordSearchRequest request = KeywordSearchRequest.of("concurrent", expectedCount); + request.setKnowledgeId("concurrent-kb"); + Assert.assertEquals(expectedCount, searcher.searchDocuments(request).size()); + } finally { + executor.shutdownNow(); + } + } + + /** + * 验证相同文档 ID 重试写入时执行覆盖而不会保留重复旧索引。 + * + * @throws Exception 临时目录或 Lucene 资源操作失败时抛出 + */ + @Test + public void shouldUpsertDocumentByIdOnRetry() throws Exception { + Path tempDir = Files.createTempDirectory("lucene-searcher-upsert-test"); + LuceneConfig config = new LuceneConfig(); + config.setIndexDirPath(tempDir.toString()); + try (LuceneSearcher searcher = new LuceneSearcher(config)) { + Document document = new Document(); + document.setId("retry-id"); + document.setContent("oldkeyword"); + Assert.assertTrue(searcher.addDocument(document)); + + document.setContent("newkeyword"); + Assert.assertTrue(searcher.addDocument(document)); + + Assert.assertTrue(searcher.searchDocuments("oldkeyword", 10).isEmpty()); + Assert.assertEquals(1, searcher.searchDocuments("newkeyword", 10).size()); + } + } + + /** + * 验证多个文档可在一次提交中批量删除。 + * + * @throws Exception 临时目录或 Lucene 资源操作失败时抛出 + */ + @Test + public void shouldDeleteDocumentsInSingleBatch() throws Exception { + Path tempDir = Files.createTempDirectory("lucene-searcher-delete-test"); + LuceneConfig config = new LuceneConfig(); + config.setIndexDirPath(tempDir.toString()); + try (LuceneSearcher searcher = new LuceneSearcher(config)) { + Document first = new Document(); + first.setId("delete-first"); + first.setContent("batchdelete"); + Document second = new Document(); + second.setId("delete-second"); + second.setContent("batchdelete"); + + Assert.assertTrue(searcher.addDocuments(List.of(first, second))); + Assert.assertTrue(searcher.deleteDocuments(List.of(first.getId(), second.getId()))); + Assert.assertTrue(searcher.searchDocuments("batchdelete", 10).isEmpty()); + } } } diff --git a/easy-agents-search-engine/easy-agents-search-engine-service/src/main/java/com/easyagents/search/engine/service/DocumentSearcher.java b/easy-agents-search-engine/easy-agents-search-engine-service/src/main/java/com/easyagents/search/engine/service/DocumentSearcher.java index 00e84c6..ab248fc 100644 --- a/easy-agents-search-engine/easy-agents-search-engine-service/src/main/java/com/easyagents/search/engine/service/DocumentSearcher.java +++ b/easy-agents-search-engine/easy-agents-search-engine-service/src/main/java/com/easyagents/search/engine/service/DocumentSearcher.java @@ -17,23 +17,106 @@ package com.easyagents.search.engine.service; import com.easyagents.core.document.Document; +import java.util.Collection; import java.util.List; +/** + * 关键词搜索引擎统一接口。 + */ public interface DocumentSearcher { + /** + * 写入单个文档。 + * + * @param document 待写入文档 + * @return 写入成功时返回 {@code true} + */ boolean addDocument(Document document); + /** + * 批量写入文档。 + * + *

默认实现保持现有搜索引擎兼容性;支持原生批量写入的实现应覆盖该方法。

+ * + * @param documents 待写入文档 + * @return 全部文档写入成功时返回 {@code true} + */ + default boolean addDocuments(List documents) { + if (documents == null || documents.isEmpty()) { + return false; + } + for (Document document : documents) { + if (!addDocument(document)) { + return false; + } + } + return true; + } + + /** + * 删除指定文档。 + * + * @param id 文档 ID + * @return 删除成功时返回 {@code true} + */ boolean deleteDocument(Object id); + /** + * 批量删除指定文档。 + * + *

默认实现保持现有搜索引擎兼容性,并确保所有文档都会尝试删除; + * 支持原生批量删除的实现应覆盖该方法。

+ * + * @param ids 文档 ID 集合 + * @return 全部文档删除成功时返回 {@code true} + */ + default boolean deleteDocuments(Collection ids) { + if (ids == null || ids.isEmpty()) { + return false; + } + boolean success = true; + for (Object id : ids) { + if (!deleteDocument(id)) { + success = false; + } + } + return success; + } + + /** + * 更新指定文档。 + * + * @param document 待更新文档 + * @return 更新成功时返回 {@code true} + */ boolean updateDocument(Document document); + /** + * 使用默认返回数量搜索文档。 + * + * @param keyword 搜索关键词 + * @return 命中文档 + */ default List searchDocuments(String keyword) { return searchDocuments(KeywordSearchRequest.of(keyword, 10)); } + /** + * 搜索指定数量的文档。 + * + * @param keyword 搜索关键词 + * @param count 最大返回数量 + * @return 命中文档 + */ default List searchDocuments(String keyword, int count) { return searchDocuments(KeywordSearchRequest.of(keyword, count)); } + /** + * 按请求条件搜索文档。 + * + * @param request 搜索请求 + * @return 命中文档 + */ List searchDocuments(KeywordSearchRequest request); } From 8d8d77ffdaa4d8aa64c77708d68b3e4f6bd49f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Sun, 9 Aug 2026 21:24:40 +0800 Subject: [PATCH 24/33] =?UTF-8?q?feat:=20=E5=AE=8C=E5=96=84=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E6=B5=81=E5=AE=9E=E4=BE=8B=E7=8A=B6=E6=80=81=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E4=B8=8E=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 增加暂停态原子恢复守卫,避免重复恢复覆盖实例状态 - 支持从实例定义快照读取节点名称 --- .../com/easyagents/flow/core/chain/Chain.java | 40 ++++++++ .../core/chain/runtime/ChainExecutor.java | 76 +++++++++++++++ .../ChainExecutorInstanceMetadataTest.java | 68 +++++++++++++ .../flow/core/test/ChainResumeGuardTest.java | 96 +++++++++++++++++++ 4 files changed, 280 insertions(+) create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorInstanceMetadataTest.java create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainResumeGuardTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java index 9fc1fb1..b04db36 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/Chain.java @@ -1732,7 +1732,47 @@ public class Chain { } + /** + * 仅在工作流处于暂停状态时恢复执行。 + * + *

状态判断与恢复动作在同一个实例锁内完成,避免并发恢复请求重复注入变量 + * 或把终态实例重新改为运行中。

+ * + * @param variables 恢复时注入的变量 + * @return 本次是否完成了暂停态到运行态的转换 + */ + public boolean resumeIfSuspended(Map variables) { + return executeWithLock( + stateInstanceId, + 10L, + TimeUnit.SECONDS, + () -> { + ChainState current = + chainStateRepository.load(stateInstanceId); + if (current == null + || current.getStatus() != ChainStatus.SUSPEND) { + return false; + } + resumeSuspended(variables); + return true; + }); + } + + /** + * 恢复暂停中的工作流。 + * + * @param variables 恢复时注入的变量 + */ public void resume(Map variables) { + resumeIfSuspended(variables); + } + + /** + * 在调用方持有实例锁且已确认暂停状态后执行恢复动作。 + * + * @param variables 恢复时注入的变量 + */ + private void resumeSuspended(Map variables) { ChainState newState = updateStateSafely(state -> { if (variables != null) { state.getMemory().putAll(variables); diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java index c2491e2..a752ada 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java @@ -896,6 +896,82 @@ public class ChainExecutor { chain.resume(variables); } + /** + * 仅在工作流实例处于暂停状态时恢复执行。 + * + *

状态判断和恢复由 {@link Chain} 在同一个实例锁内完成,可安全处理并发恢复请求。

+ * + * @param stateInstanceId 工作流实例 ID + * @param variables 恢复时注入的变量 + * @return 本次是否完成了暂停态到运行态的转换 + */ + public boolean resumeAsyncIfSuspended( + String stateInstanceId, + Map variables) { + ChainState state = chainStateRepository.load(stateInstanceId); + if (state == null) { + return false; + } + ChainDefinition definition = getDefinitionForInstance(state); + if (definition == null) { + return false; + } + Chain chain = configureChain( + definition, + state.getInstanceId(), + state); + return chain.resumeIfSuspended(variables); + } + + /** + * 获取工作流实例启动时定义快照中的节点名称。 + * + * @param stateInstanceId 工作流实例 ID + * @return 按定义顺序排列的节点 ID 与名称;实例或定义不存在时返回空映射 + */ + public Map getInstanceNodeNames( + String stateInstanceId) { + ChainState state = chainStateRepository.load(stateInstanceId); + if (state == null) { + return Collections.emptyMap(); + } + return getInstanceNodeNames(state); + } + + /** + * 使用调用方已经加载的状态获取实例定义快照中的节点名称。 + * + * @param state 已加载的工作流状态 + * @return 按定义顺序排列的节点 ID 与名称;状态或定义不存在时返回空映射 + */ + public Map getInstanceNodeNames( + ChainState state) { + if (state == null) { + return Collections.emptyMap(); + } + ChainDefinition definition = getDefinitionForInstance(state); + if (definition == null + || definition.getNodes() == null + || definition.getNodes().isEmpty()) { + return Collections.emptyMap(); + } + Map nodeNames = new LinkedHashMap<>(); + for (Node node : definition.getNodes()) { + if (node == null + || node.getId() == null + || node.getId().isBlank()) { + continue; + } + String nodeName = node.getName(); + nodeNames.put( + node.getId(), + nodeName == null || nodeName.isBlank() + ? node.getId() + : nodeName); + } + return Collections.unmodifiableMap(nodeNames); + } + private Chain createChain(String definitionId) { ChainDefinition definition = definitionRepository.getChainDefinitionById(definitionId); diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorInstanceMetadataTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorInstanceMetadataTest.java new file mode 100644 index 0000000..9be3d60 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorInstanceMetadataTest.java @@ -0,0 +1,68 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import com.easyagents.flow.core.chain.runtime.ChainExecutor; +import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore; +import com.easyagents.flow.core.chain.runtime.TriggerScheduler; +import com.easyagents.flow.core.node.StartNode; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +/** + * {@link ChainExecutor} 实例定义元数据查询测试。 + */ +public class ChainExecutorInstanceMetadataTest { + + /** + * 验证节点名称来自实例启动时可恢复的定义快照。 + */ + @Test + public void shouldResolveNodeNamesFromInstanceDefinition() { + ChainDefinition definition = new ChainDefinition(); + definition.setId("metadata-definition"); + StartNode start = new StartNode(); + start.setId("start"); + start.setName("开始节点"); + definition.setNodes(Collections.singletonList(start)); + definition.setEdges(Collections.emptyList()); + + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + stateRepository.create("metadata-instance") + .setChainDefinitionId(definition.getId()); + ScheduledExecutorService schedulerPool = + Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = + Executors.newSingleThreadExecutor(); + TriggerScheduler scheduler = new TriggerScheduler( + new InMemoryTriggerStore(), + schedulerPool, + workerPool, + 1_000L); + ChainExecutor executor = new ChainExecutor( + ignored -> definition, + stateRepository, + new InMemoryNodeStateRepository(), + scheduler); + + try { + Map nodeNames = + executor.getInstanceNodeNames( + "metadata-instance"); + + Assert.assertEquals( + Map.of("start", "开始节点"), + nodeNames); + } finally { + scheduler.shutdown(); + } + } +} diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainResumeGuardTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainResumeGuardTest.java new file mode 100644 index 0000000..6013c34 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainResumeGuardTest.java @@ -0,0 +1,96 @@ +package com.easyagents.flow.core.test; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.EventManager; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +/** + * {@link Chain} 暂停恢复状态守卫测试。 + */ +public class ChainResumeGuardTest { + + /** + * 验证只有暂停中的实例可以恢复,重复恢复不会再次注入变量。 + */ + @Test + public void shouldResumeOnlyOnceFromSuspendedState() { + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + Chain chain = createChain(stateRepository, "resume-once"); + chain.suspend(); + + boolean resumed = chain.resumeIfSuspended( + Map.of("approved", true)); + boolean resumedAgain = chain.resumeIfSuspended( + Map.of("unexpected", true)); + + Assert.assertTrue(resumed); + Assert.assertFalse(resumedAgain); + Assert.assertEquals( + ChainStatus.RUNNING, + stateRepository.load("resume-once").getStatus()); + Assert.assertEquals( + Boolean.TRUE, + stateRepository.load("resume-once") + .getMemory() + .get("approved")); + Assert.assertFalse( + stateRepository.load("resume-once") + .getMemory() + .containsKey("unexpected")); + } + + /** + * 验证成功终态不会被恢复操作改回运行中。 + */ + @Test + public void shouldKeepTerminalStateUnchanged() { + InMemoryChainStateRepository stateRepository = + new InMemoryChainStateRepository(); + Chain chain = createChain(stateRepository, "resume-terminal"); + stateRepository.load("resume-terminal") + .setStatus(ChainStatus.SUCCEEDED); + + boolean resumed = chain.resumeIfSuspended( + Map.of("unexpected", true)); + + Assert.assertFalse(resumed); + Assert.assertEquals( + ChainStatus.SUCCEEDED, + stateRepository.load("resume-terminal").getStatus()); + Assert.assertFalse( + stateRepository.load("resume-terminal") + .getMemory() + .containsKey("unexpected")); + } + + /** + * 创建使用进程内状态仓储的最小工作流。 + * + * @param stateRepository 状态仓储 + * @param instanceId 工作流实例 ID + * @return 已配置工作流 + */ + private Chain createChain( + InMemoryChainStateRepository stateRepository, + String instanceId) { + ChainDefinition definition = new ChainDefinition(); + definition.setId("resume-definition"); + definition.setNodes(Collections.emptyList()); + definition.setEdges(Collections.emptyList()); + Chain chain = new Chain(definition, instanceId); + chain.setChainStateRepository(stateRepository); + chain.setNodeStateRepository( + new InMemoryNodeStateRepository()); + chain.setEventManager(new EventManager()); + return chain; + } +} From 857fe7caf8ae69acea7152c44c06c31dcc80e4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Tue, 11 Aug 2026 20:56:05 +0800 Subject: [PATCH 25/33] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20OpenAI=20=E6=B6=88=E6=81=AF=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E5=9D=97=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增标准字符串与文本内容块数组两种序列化模式 - 统一处理各角色消息并保留多模态及工具调用结构 - 补充默认模式与内容块模式测试 --- .../core/model/chat/ChatConfig.java | 32 +++++ .../model/chat/ChatMessageContentFormat.java | 28 +++++ .../client/OpenAIChatMessageSerializer.java | 29 ++++- .../OpenAIChatMessageSerializerTest.java | 116 ++++++++++++++++++ 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatMessageContentFormat.java diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatConfig.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatConfig.java index 55b8445..4e4bea5 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatConfig.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatConfig.java @@ -27,6 +27,9 @@ public class ChatConfig extends BaseModelConfig { protected Boolean supportToolMessage; protected Boolean supportThinking; + /** OpenAI-compatible 消息 content 的序列化格式。 */ + protected ChatMessageContentFormat messageContentFormat = ChatMessageContentFormat.STANDARD; + // 在调用工具的时候,是否需要推理结果作为 reasoning_content 传给大模型, 比如 Deepseek // 参考文档: https://api-docs.deepseek.com/zh-cn/guides/thinking_mode#%E5%B7%A5%E5%85%B7%E8%B0%83%E7%94%A8 protected Boolean needReasoningContentForToolMessage; @@ -135,6 +138,35 @@ public class ChatConfig extends BaseModelConfig { return supportThinking == null || supportThinking; } + /** + * 获取消息 content 的序列化格式。 + * + * @return 消息 content 格式 + */ + public ChatMessageContentFormat getMessageContentFormat() { + return messageContentFormat; + } + + /** + * 设置消息 content 的序列化格式。 + * + * @param messageContentFormat 消息 content 格式;null 时回退为标准格式 + */ + public void setMessageContentFormat(ChatMessageContentFormat messageContentFormat) { + this.messageContentFormat = messageContentFormat == null + ? ChatMessageContentFormat.STANDARD + : messageContentFormat; + } + + /** + * 判断是否需要将纯文本 content 序列化为内容块数组。 + * + * @return 配置为内容块数组时返回 true + */ + public boolean isTextPartsMessageContent() { + return messageContentFormat == ChatMessageContentFormat.TEXT_PARTS; + } + public Boolean getNeedReasoningContentForToolMessage() { return needReasoningContentForToolMessage; } diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatMessageContentFormat.java b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatMessageContentFormat.java new file mode 100644 index 0000000..4a84c64 --- /dev/null +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/chat/ChatMessageContentFormat.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2026, Easy-Agents (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.easyagents.core.model.chat; + +/** + * OpenAI-compatible 消息 content 的序列化格式。 + */ +public enum ChatMessageContentFormat { + + /** 保持供应商默认格式,纯文本 content 使用字符串。 */ + STANDARD, + + /** 将各角色的纯文本 content 统一序列化为文本内容块数组。 */ + TEXT_PARTS +} diff --git a/easy-agents-core/src/main/java/com/easyagents/core/model/client/OpenAIChatMessageSerializer.java b/easy-agents-core/src/main/java/com/easyagents/core/model/client/OpenAIChatMessageSerializer.java index 88fe800..82b4750 100644 --- a/easy-agents-core/src/main/java/com/easyagents/core/model/client/OpenAIChatMessageSerializer.java +++ b/easy-agents-core/src/main/java/com/easyagents/core/model/client/OpenAIChatMessageSerializer.java @@ -58,11 +58,39 @@ public class OpenAIChatMessageSerializer implements ChatMessageSerializer { } else if (message instanceof ToolMessage) { buildToolMessageObject(objectMap, (ToolMessage) message, config); } + normalizeMessageContent(objectMap, config); messageList.add(objectMap); }); return messageList; } + /** + * 根据模型配置将纯文本 content 规范为 OpenAI 文本内容块数组。 + * 已经是多模态内容块数组的 content 保持不变。 + * + * @param objectMap 已完成角色字段构建的消息 + * @param config 模型配置 + */ + protected void normalizeMessageContent(Map objectMap, ChatConfig config) { + if (config == null + || !config.isTextPartsMessageContent() + || !objectMap.containsKey("content")) { + return; + } + + Object content = objectMap.get("content"); + if (content instanceof List) { + return; + } + if (content == null || content instanceof String) { + String text = content == null ? "" : (String) content; + objectMap.put("content", List.of(Maps.of("type", "text").set("text", text))); + return; + } + throw new IllegalStateException( + "Unsupported OpenAI message content type: " + content.getClass().getName()); + } + protected void buildToolMessageObject(Map objectMap, ToolMessage message, ChatConfig config) { if (config.isSupportToolMessage()) { objectMap.put("role", "tool"); @@ -289,4 +317,3 @@ public class OpenAIChatMessageSerializer implements ChatMessageSerializer { } } } - diff --git a/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java b/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java index 6fa0b24..0d93579 100644 --- a/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java +++ b/easy-agents-core/src/test/java/com/easyagents/core/test/model/client/OpenAIChatMessageSerializerTest.java @@ -1,7 +1,12 @@ package com.easyagents.core.test.model.client; +import com.easyagents.core.message.AiMessage; +import com.easyagents.core.message.SystemMessage; +import com.easyagents.core.message.ToolCall; +import com.easyagents.core.message.ToolMessage; import com.easyagents.core.message.UserMessage; import com.easyagents.core.model.chat.ChatConfig; +import com.easyagents.core.model.chat.ChatMessageContentFormat; import com.easyagents.core.model.client.OpenAIChatMessageSerializer; import org.junit.Assert; import org.junit.Test; @@ -14,6 +19,50 @@ import java.util.Map; */ public class OpenAIChatMessageSerializerTest { + /** + * 验证标准模式继续使用原有纯文本字符串格式。 + */ + @Test + public void shouldKeepStringContentInStandardMode() { + ToolMessage toolMessage = toolMessage("call-1", "工具结果"); + + List> messages = new OpenAIChatMessageSerializer() + .serializeMessages(List.of( + SystemMessage.of("系统提示"), + new UserMessage("用户问题"), + new AiMessage("助手回答"), + toolMessage + ), new ChatConfig()); + + Assert.assertEquals("系统提示", messages.get(0).get("content")); + Assert.assertEquals("用户问题", messages.get(1).get("content")); + Assert.assertEquals("助手回答", messages.get(2).get("content")); + Assert.assertEquals("工具结果", messages.get(3).get("content")); + } + + /** + * 验证内容块模式会转换全部纯文本消息角色。 + */ + @Test + public void shouldSerializeAllTextRolesAsContentParts() { + ChatConfig config = textPartsConfig(); + ToolMessage toolMessage = toolMessage("call-1", "工具结果"); + + List> messages = new OpenAIChatMessageSerializer() + .serializeMessages(List.of( + SystemMessage.of("系统提示"), + new UserMessage("用户问题"), + new AiMessage("助手回答"), + toolMessage + ), config); + + assertTextPart(messages.get(0), "系统提示"); + assertTextPart(messages.get(1), "用户问题"); + assertTextPart(messages.get(2), "助手回答"); + assertTextPart(messages.get(3), "工具结果"); + Assert.assertEquals("call-1", messages.get(3).get("tool_call_id")); + } + /** * 验证 Data URI 会写入标准的 image_url.url 字段。 */ @@ -33,4 +82,71 @@ public class OpenAIChatMessageSerializerTest { Assert.assertEquals("image_url", imageContent.get("type")); Assert.assertEquals(dataUri, imageUrl.get("url")); } + + /** + * 验证内容块模式保留多模态数组和工具调用结构字段。 + */ + @Test + public void shouldPreserveStructuredFieldsInTextPartsMode() { + String dataUri = "data:image/png;base64,AQID"; + UserMessage userMessage = new UserMessage("识别图片"); + userMessage.addImageUrl(dataUri); + AiMessage assistantMessage = new AiMessage(null); + assistantMessage.setReasoningContent("先分析"); + assistantMessage.setToolCalls(List.of( + new ToolCall("call-1", "image_search", "{\"query\":\"license\"}"))); + ToolMessage toolMessage = toolMessage("call-1", "工具结果"); + + List> messages = new OpenAIChatMessageSerializer() + .serializeMessages(List.of(userMessage, assistantMessage, toolMessage), textPartsConfig()); + + List userContent = (List) messages.get(0).get("content"); + Assert.assertEquals(2, userContent.size()); + Assert.assertEquals("image_url", ((Map) userContent.get(1)).get("type")); + assertTextPart(messages.get(1), ""); + Assert.assertEquals("先分析", messages.get(1).get("reasoning_content")); + Assert.assertTrue(messages.get(1).containsKey("tool_calls")); + assertTextPart(messages.get(2), "工具结果"); + Assert.assertEquals("call-1", messages.get(2).get("tool_call_id")); + } + + /** + * 创建内容块数组模式配置。 + * + * @return 内容块数组模式配置 + */ + private ChatConfig textPartsConfig() { + ChatConfig config = new ChatConfig(); + config.setMessageContentFormat(ChatMessageContentFormat.TEXT_PARTS); + config.setNeedReasoningContentForToolMessage(Boolean.TRUE); + return config; + } + + /** + * 创建工具结果消息。 + * + * @param toolCallId 工具调用标识 + * @param content 工具结果文本 + * @return 工具结果消息 + */ + private ToolMessage toolMessage(String toolCallId, String content) { + ToolMessage message = new ToolMessage(); + message.setToolCallId(toolCallId); + message.setContent(content); + return message; + } + + /** + * 断言消息 content 只包含指定文本内容块。 + * + * @param message 已序列化消息 + * @param expectedText 预期文本 + */ + private void assertTextPart(Map message, String expectedText) { + List content = (List) message.get("content"); + Assert.assertEquals(1, content.size()); + Map textPart = (Map) content.get(0); + Assert.assertEquals("text", textPart.get("type")); + Assert.assertEquals(expectedText, textPart.get("text")); + } } From b313523aba7d4db316eebc892c1b824878be676a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Tue, 11 Aug 2026 21:41:55 +0800 Subject: [PATCH 26/33] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E6=A8=A1=E6=9D=BF=E4=B8=AD=E6=96=87=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E6=B8=B2=E6=9F=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 开启 Enjoy 中文表达式支持 - 补充中文、英文及上游引用回归测试 --- .../flow/core/node/TemplateNode.java | 2 + .../flow/core/node/TemplateNodeTest.java | 125 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 easy-agents-flow/src/test/java/com/easyagents/flow/core/node/TemplateNodeTest.java diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java index d6006cb..2aa57e3 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/node/TemplateNode.java @@ -34,6 +34,8 @@ public class TemplateNode extends BaseNode { private String template; static { + // Enjoy 默认仅识别 ASCII 变量名,工作流参数需要支持中文名称。 + Engine.setChineseExpression(true); engine = Engine.create("template", e -> { e.addSharedStaticMethod(StringUtil.class); }); diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/node/TemplateNodeTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/node/TemplateNodeTest.java new file mode 100644 index 0000000..c584d57 --- /dev/null +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/node/TemplateNodeTest.java @@ -0,0 +1,125 @@ +package com.easyagents.flow.core.node; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainDefinition; +import com.easyagents.flow.core.chain.ChainState; +import com.easyagents.flow.core.chain.Parameter; +import com.easyagents.flow.core.chain.RefType; +import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository; +import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +/** + * 内容模板节点变量渲染回归测试。 + */ +public class TemplateNodeTest { + + /** + * 验证模板可以使用中文参数名称。 + */ + @Test + public void shouldRenderChineseParameterNames() { + TemplateNode node = templateNode( + "#(申请人)\n\n#(被申请人)", + "申请人", + "被申请人"); + Chain chain = chain(Map.of( + "申请人", "申请内容", + "被申请人", "答辩内容")); + + Map result = node.execute(chain); + + Assert.assertEquals( + "申请内容\n\n答辩内容", + result.get("finalContent")); + } + + /** + * 验证开启中文表达式后继续兼容英文参数名称。 + */ + @Test + public void shouldKeepRenderingEnglishParameterNames() { + TemplateNode node = templateNode( + "#(applicant)\n\n#(respondent)", + "applicant", + "respondent"); + Chain chain = chain(Map.of( + "applicant", "申请内容", + "respondent", "答辩内容")); + + Map result = node.execute(chain); + + Assert.assertEquals( + "申请内容\n\n答辩内容", + result.get("finalContent")); + } + + /** + * 验证自动模板变量可以通过引用读取上游节点输出。 + */ + @Test + public void shouldRenderManagedUpstreamReference() { + String parameterName = "ref_node__llm_2e_output"; + TemplateNode node = templateNode( + "模型输出:#(" + parameterName + ")"); + Parameter parameter = new Parameter(parameterName); + parameter.setRefType(RefType.REF); + parameter.setRef("node_llm.output"); + node.setParameters(Collections.singletonList(parameter)); + Chain chain = chain(Map.of( + "node_llm.output", "回答内容")); + + Map result = node.execute(chain); + + Assert.assertEquals( + "模型输出:回答内容", + result.get("finalContent")); + } + + /** + * 创建指定输入参数的内容模板节点。 + * + * @param template 模板内容 + * @param parameterNames 输入参数名称 + * @return 内容模板节点 + */ + private TemplateNode templateNode( + String template, + String... parameterNames) { + TemplateNode node = new TemplateNode(); + node.setId("template-node"); + node.setName("内容模板"); + node.setTemplate(template); + node.setParameters(Arrays.stream(parameterNames) + .map(Parameter::new) + .toList()); + node.setOutputDefs(Collections.singletonList( + new Parameter("finalContent"))); + return node; + } + + /** + * 创建带初始化状态和输入变量的工作流。 + * + * @param inputs 工作流输入 + * @return 工作流 + */ + private Chain chain(Map inputs) { + Chain chain = new Chain( + new ChainDefinition(), + "template-node-" + UUID.randomUUID()); + chain.setChainStateRepository( + new InMemoryChainStateRepository()); + chain.setNodeStateRepository( + new InMemoryNodeStateRepository()); + ChainState state = chain.initializeState(); + state.getMemory().putAll(inputs); + return chain; + } +} From a34ca9271ef3273e4373be7e050e2ba0de47e572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 14 Aug 2026 18:51:43 +0800 Subject: [PATCH 27/33] =?UTF-8?q?refactor:=20=E7=B2=BE=E7=AE=80=E6=A0=87?= =?UTF-8?q?=E5=87=86=20Skill=20=E5=8C=85=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 统一使用 SKILL.md 与通用资源表达 - 删除仓储、专用资源类型和低价值兼容接口 - 保留安全 ZIP 编解码、校验和内容存储能力 --- easy-agents-skill/README.md | 19 +- .../skill/codec/SkillPackageCodec.java | 59 ---- .../skill/codec/SkillPackageWriteResult.java | 11 - .../skill/codec/ZipSkillPackageCodec.java | 90 ++---- .../skill/factory/SkillFactory.java | 77 +----- .../com/easyagents/skill/model/Skill.java | 160 +---------- .../easyagents/skill/model/SkillAsset.java | 148 ---------- .../skill/model/SkillDescriptor.java | 111 -------- .../skill/model/SkillReference.java | 129 --------- .../easyagents/skill/model/SkillResource.java | 19 -- .../easyagents/skill/model/SkillScript.java | 129 --------- .../skill/model/SkillScriptLanguage.java | 50 ---- .../skill/repository/SkillRepository.java | 58 ---- .../memory/InMemorySkillRepository.java | 214 --------------- .../skill/store/SkillContentStage.java | 28 +- .../skill/store/SkillContentStore.java | 146 +--------- .../file/TemporaryFileSkillContentStore.java | 34 +-- .../memory/InMemorySkillContentStore.java | 210 -------------- .../easyagents/skill/util/SkillResources.java | 156 ++--------- .../skill/validation/SkillValidator.java | 65 ----- .../defaults/DefaultSkillValidator.java | 70 ++--- .../skill/codec/ZipSkillPackageCodecTest.java | 257 +++++++----------- .../memory/InMemorySkillRepositoryTest.java | 118 -------- .../TemporaryFileSkillContentStoreTest.java | 2 - .../memory/InMemorySkillContentStore.java | 124 +++++++++ .../memory/InMemorySkillContentStoreTest.java | 83 ------ .../skill/util/SkillResourcesTest.java | 15 +- .../defaults/DefaultSkillValidatorTest.java | 79 ++---- 28 files changed, 366 insertions(+), 2295 deletions(-) delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java delete mode 100644 easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java delete mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java create mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java delete mode 100644 easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java diff --git a/easy-agents-skill/README.md b/easy-agents-skill/README.md index 08f0cd8..3c916ab 100644 --- a/easy-agents-skill/README.md +++ b/easy-agents-skill/README.md @@ -14,6 +14,13 @@ Codec 支持根目录单 Skill、单目录 Skill 和多目录 Skill 三种输入 `SKILL.md` 使用 YAML frontmatter 与 Markdown 正文。未知字段、嵌套 Map/List、布尔值和数字会保留;语义校验通过结构化 issue 返回路径、行列、错误码与修复建议。 +模块内的可移植模型只有两层: + +- `SkillDocument`:`SKILL.md` 原文、frontmatter 和 Markdown 正文 +- `SkillResource`:除 `SKILL.md` 外的任意安全相对路径文件 + +Skill 的数据库主键、分类、权限、发布和审批属于上层技能库,不进入标准包模型。 + ## 推荐调用方式 无参 `ZipSkillPackageCodec` 使用实例级临时文件存储,适合一次性导入导出。它拥有临时目录,必须关闭: @@ -41,10 +48,10 @@ codec.encode(result.getSkillPackage(), outputStream, writeOptions); 校验通过 `SkillValidationMode` 区分两个明确上下文: -- `DRAFT_IMPORT`:ZIP 导入和兼容预检使用;历史下划线名称保留为 warning,允许先进入草稿修复。 +- `DRAFT_IMPORT`:ZIP 导入和草稿编辑使用;可修复的命名问题返回 warning。 - `STANDARD`:正式新建、发布校验和标准 ZIP 导出使用;下划线名称等互操作问题作为 error。 -`SkillFactory.createStrict`、`SkillFactory.createWithResourcesStrict`、`DefaultSkillValidator.validate` 与 `ZipSkillPackageCodec.encode` 执行 `STANDARD` 校验。为保持旧调用兼容,`SkillFactory.create` 仍可构建导入草稿,原有 `validateReport(skill)` 与 `validateReport(skill, limits)` 继续使用 `DRAFT_IMPORT`;新调用方需要显式上下文时使用三参数 `validateReport`。 +`DefaultSkillValidator.validate` 与 `ZipSkillPackageCodec.encode` 执行 `STANDARD` 校验。`SkillFactory.create` 构建草稿;需要指定校验上下文时调用三参数 `validateReport`。 ## 安全边界 @@ -59,14 +66,14 @@ codec.encode(result.getSkillPackage(), outputStream, writeOptions); 限额通过 `SkillPackageLimits` 配置,并由 `SkillPackageReadOptions`、`SkillPackageWriteOptions` 传入单次操作。 -## 从旧接口迁移 +## 编解码结果 -`importZip(InputStream)` 为兼容入口,现已废弃。新调用方应使用 `decode`,以获得: +`decode` 返回: - 包布局 `SkillPackageLayout` - 标准化 `SkillPackage` - 包哈希 - 聚合校验报告 -- `STRICT` 或 `REPORT_ONLY` 读取模式 +- `COMMIT_ON_VALID` 或 `REPORT_ONLY` 读取模式 -写出统一使用 `encode`。自定义校验器是附加业务校验,不能替代 Codec 内置的标准安全校验。 +写出统一使用 `encode`。Codec 始终执行内置标准与安全校验。 diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java deleted file mode 100644 index 6da742d..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageCodec.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.easyagents.skill.codec; - -import com.easyagents.skill.exception.SkillPackageException; -import com.easyagents.skill.model.Skill; -import com.easyagents.skill.model.SkillPackage; -import com.easyagents.skill.model.SkillPackageLayout; -import com.easyagents.skill.validation.SkillValidationReport; - -import java.io.InputStream; -import java.io.OutputStream; -import java.util.List; - -/** - * Skill 包双向流式编解码接口。 - */ -public interface SkillPackageCodec { - - /** - * 从 ZIP 输入流导入 Skill。 - * - * @param inputStream ZIP 输入流 - * @return Skill 列表 - * @throws SkillPackageException ZIP 结构、内容或安全校验失败 - * @deprecated 请使用 {@link #decode(InputStream, SkillPackageReadOptions)} 获取包形态、hash 和诊断。 - */ - @Deprecated - List importZip(InputStream inputStream); - - /** - * 解码 Skill ZIP。 - * - *

兼容默认实现委托旧导入接口;正式 Codec 应覆盖。

- * - * @param inputStream ZIP 输入流 - * @param options 读取选项 - * @return 解码结果 - * @throws SkillPackageException ZIP 结构、内容、安全校验或资源存储失败 - */ - default SkillPackageReadResult decode(InputStream inputStream, SkillPackageReadOptions options) { - List skills = importZip(inputStream); - SkillPackageLayout layout = skills.size() > 1 - ? SkillPackageLayout.MULTI_DIRECTORY : SkillPackageLayout.SINGLE_DIRECTORY; - return new SkillPackageReadResult(new SkillPackage(layout, skills), new SkillValidationReport(), null); - } - - /** - * 将 Skill 包编码为标准 ZIP。 - * - * @param skillPackage Skill 包 - * @param outputStream 输出流,不由本方法关闭 - * @param options 写出选项 - * @return 编码结果 - * @throws SkillPackageException Skill 包不合法、资源不可读或 ZIP 写出失败 - */ - default SkillPackageWriteResult encode(SkillPackage skillPackage, OutputStream outputStream, - SkillPackageWriteOptions options) { - throw new SkillPackageException("This SkillPackageCodec does not support encoding."); - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java index 35e0b96..f3bdca2 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/SkillPackageWriteResult.java @@ -12,17 +12,6 @@ public final class SkillPackageWriteResult { private final int entryCount; private final SkillPackageLayout layout; - /** - * 创建编码结果。 - * - * @param packageHash 输出 ZIP SHA-256 - * @param size 输出字节数 - * @param entryCount 输出 entry 数 - */ - public SkillPackageWriteResult(String packageHash, long size, int entryCount) { - this(packageHash, size, entryCount, SkillPackageLayout.SINGLE_DIRECTORY); - } - /** * 创建带包形态的编码结果。 * diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java index e7abef7..8f14d13 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java @@ -4,7 +4,6 @@ import com.easyagents.skill.exception.SkillPackageException; import com.easyagents.skill.exception.SkillValidationException; import com.easyagents.skill.model.Skill; import com.easyagents.skill.model.SkillDocument; -import com.easyagents.skill.model.SkillMetadata; import com.easyagents.skill.model.SkillPackage; import com.easyagents.skill.model.SkillPackageLayout; import com.easyagents.skill.model.SkillPackageLimits; @@ -22,7 +21,6 @@ import com.easyagents.skill.validation.SkillValidationIssue; import com.easyagents.skill.validation.SkillValidationMode; import com.easyagents.skill.validation.SkillValidationReport; import com.easyagents.skill.validation.SkillValidationSeverity; -import com.easyagents.skill.validation.SkillValidator; import com.easyagents.skill.validation.defaults.DefaultSkillValidator; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; @@ -62,7 +60,7 @@ import java.util.zip.ZipOutputStream; /** * 标准 Skill ZIP 的安全双向流式 Codec。 */ -public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { +public class ZipSkillPackageCodec implements AutoCloseable { private static final String DEFAULT_MEDIA_TYPE = "application/octet-stream"; private static final int CENTRAL_DIRECTORY_HEADER_SIZE = 46; @@ -79,7 +77,6 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { private static final int UINT16_MAX = 0xFFFF; private final SkillContentStore contentStore; - private final SkillValidator additionalValidator; private final boolean ownsContentStore; /** @@ -88,7 +85,7 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { *

该实例拥有默认内容存储,使用完毕后应调用 {@link #close()} 清理临时内容。

*/ public ZipSkillPackageCodec() { - this(new TemporaryFileSkillContentStore(), null, true); + this(new TemporaryFileSkillContentStore(), true); } /** @@ -97,36 +94,17 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { * @param contentStore 二进制内容存储 */ public ZipSkillPackageCodec(SkillContentStore contentStore) { - this(contentStore, null, false); + this(contentStore, false); } - /** - * 创建 ZIP Codec。 - * - * @param contentStore 二进制内容存储 - * @param validator Skill 聚合校验器 - */ - public ZipSkillPackageCodec(SkillContentStore contentStore, SkillValidator validator) { - this(contentStore, requireValidator(validator), false); - } - - private ZipSkillPackageCodec(SkillContentStore contentStore, SkillValidator validator, - boolean ownsContentStore) { + private ZipSkillPackageCodec(SkillContentStore contentStore, boolean ownsContentStore) { if (contentStore == null) { throw new SkillPackageException("Skill content store is required."); } this.contentStore = contentStore; - this.additionalValidator = validator; this.ownsContentStore = ownsContentStore; } - private static SkillValidator requireValidator(SkillValidator validator) { - if (validator == null) { - throw new SkillPackageException("Skill validator is required."); - } - return validator; - } - /** * 关闭 Codec 自有的默认临时内容存储。 * @@ -139,19 +117,6 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { } } - /** - * 使用兼容入口导入 ZIP。 - * - * @param inputStream ZIP 输入流 - * @return Skill 列表 - * @deprecated 请使用 {@link #decode(InputStream, SkillPackageReadOptions)}。 - */ - @Deprecated - @Override - public List importZip(InputStream inputStream) { - return decode(inputStream, SkillPackageReadOptions.defaults()).getSkillPackage().getSkills(); - } - /** * 安全解码 Skill ZIP。 * @@ -159,7 +124,6 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { * @param options 读取选项 * @return 解码结果 */ - @Override public SkillPackageReadResult decode(InputStream inputStream, SkillPackageReadOptions options) { if (inputStream == null) { throw packageError("ZIP_INPUT_REQUIRED", null, "ZIP input stream is required."); @@ -221,7 +185,6 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { * @param options 写出选项 * @return 编码结果 */ - @Override public SkillPackageWriteResult encode(SkillPackage skillPackage, OutputStream outputStream, SkillPackageWriteOptions options) { if (skillPackage == null || skillPackage.getSkills() == null || skillPackage.getSkills().isEmpty()) { @@ -370,9 +333,7 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { throw validationPackageError(e, layout == SkillPackageLayout.ROOT_SKILL ? null : group.root); } - Map frontmatter = document.getFrontmatter().getValues(); - String name = scalar(frontmatter.get("name")); - String description = scalar(frontmatter.get("description")); + String name = scalar(document.getFrontmatter().get("name")); List resources = new ArrayList<>(); for (ArchiveFile file : group.files) { @@ -385,14 +346,9 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { resources.sort(Comparator.comparing(SkillResource::getPath)); Skill skill = new Skill(); - skill.setId(null); skill.setPackageRoot(layout == SkillPackageLayout.ROOT_SKILL ? name : group.root); - skill.setName(name); - skill.setDescription(description); - skill.setMetadata(new SkillMetadata(frontmatter)); skill.setDocument(document); skill.setResources(resources); - SkillResources.refreshLegacyViews(skill); return skill; } @@ -400,7 +356,7 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { StageTracker stages) throws IOException { SkillResourceKind kind = SkillResources.classify(file.relativePath); String mediaType = detectMediaType(file.relativePath); - boolean text = SkillResources.isText(file.relativePath, kind, mediaType); + boolean text = SkillResources.isText(file.relativePath, mediaType); long singleLimit = text ? limits.getMaxTextFileBytes() : limits.getMaxBinaryFileBytes(); if (file.entry.getSize() > singleLimit) { throw packageError(text ? "TEXT_FILE_SIZE_LIMIT" : "BINARY_FILE_SIZE_LIMIT", file.fullPath, @@ -489,24 +445,16 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { } /** - * 执行 Codec 不可绕过的标准安全校验,并追加调用方业务校验。 + * 执行 Codec 不可绕过的标准安全校验。 * * @param skill Skill 聚合 * @param limits 当前读写操作限额 * @param mode 标准校验模式 - * @return 合并后的结构化报告 + * @return 结构化报告 */ private SkillValidationReport validateForCodec(Skill skill, SkillPackageLimits limits, SkillValidationMode mode) { - SkillValidationReport report = new DefaultSkillValidator(limits) - .validateReport(skill, limits, mode); - if (additionalValidator != null) { - SkillValidationReport additionalReport = additionalValidator.getClass() == DefaultSkillValidator.class - ? additionalValidator.validateReport(skill, null, mode) - : additionalValidator.validateReport(skill, limits, mode); - report.merge(additionalReport); - } - return report; + return new DefaultSkillValidator(limits).validateReport(skill, limits, mode); } private List prepareOutput(List skills, SkillPackageLimits limits) { @@ -557,7 +505,7 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { files.add(OutputFile.text(skillPath, skillContent)); totalSize = checkedOutputTotal(totalSize, skillSize, limits, skillPath); - List resources = SkillResources.canonicalResources(skill); + List resources = new ArrayList<>(skill.getResources()); resources.sort(Comparator.comparing(SkillResource::getPath)); for (SkillResource resource : resources) { String path = skill.getName() + "/" + SkillPaths.normalize(resource.getPath()); @@ -1533,22 +1481,27 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { } SkillPackageException rollbackFailure = null; for (int index = records.size() - 1; index >= 0; index--) { + StageRecord record = records.get(index); + if (record.cleaned) { + continue; + } try { - contentStore.rollback(records.get(index).stage); + contentStore.rollback(record.stage); + record.cleaned = true; } catch (RuntimeException cleanupError) { if (rollbackFailure == null) { rollbackFailure = new SkillPackageException( - "SKILL_CONTENT_ROLLBACK_ERROR", records.get(index).path, + "SKILL_CONTENT_ROLLBACK_ERROR", record.path, "Failed to rollback staged Skill package content.", cleanupError); } else { rollbackFailure.addSuppressed(cleanupError); } } } - finalized = true; if (rollbackFailure != null) { throw rollbackFailure; } + finalized = true; } private void cleanupAfterFailure(Throwable primary) { @@ -1557,17 +1510,21 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { } for (int index = records.size() - 1; index >= 0; index--) { StageRecord record = records.get(index); + if (record.cleaned) { + continue; + } try { if (record.committedRef == null) { contentStore.rollback(record.stage); } else { contentStore.release(record.committedRef); } + record.cleaned = true; } catch (RuntimeException cleanupError) { primary.addSuppressed(cleanupError); } } - finalized = true; + finalized = records.stream().allMatch(record -> record.cleaned); } } @@ -1577,6 +1534,7 @@ public class ZipSkillPackageCodec implements SkillPackageCodec, AutoCloseable { private final SkillResource resource; private final String path; private String committedRef; + private boolean cleaned; private StageRecord(SkillContentStage stage, SkillResource resource, String path) { this.stage = stage; diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java b/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java index 207816e..af4b250 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/factory/SkillFactory.java @@ -2,8 +2,6 @@ package com.easyagents.skill.factory; import com.easyagents.skill.model.*; import com.easyagents.skill.util.SkillFrontmatter; -import com.easyagents.skill.util.SkillResources; -import com.easyagents.skill.validation.defaults.DefaultSkillValidator; import java.util.List; import java.util.Map; @@ -19,92 +17,37 @@ public final class SkillFactory { /** * 基于 SKILL.md 内容创建 Skill。 * - * @param id Skill ID * @param skillContent SKILL.md 原始内容 * @return Skill 聚合 */ - public static Skill create(String id, String skillContent) { - return create(id, skillContent, null, null, null); - } - - /** - * 基于 SKILL.md 内容创建并严格校验正式标准 Skill。 - * - * @param id Skill ID - * @param skillContent SKILL.md 原始内容 - * @return 通过正式标准校验的 Skill 聚合 - */ - public static Skill createStrict(String id, String skillContent) { - Skill skill = create(id, skillContent); - new DefaultSkillValidator().validate(skill); + public static Skill create(String skillContent) { + SkillDocument document = SkillFrontmatter.parseDocument(skillContent); + Map values = document.getFrontmatter().getValues(); + requiredScalar(values, "name"); + requiredScalar(values, "description"); + Skill skill = new Skill(); + skill.setDocument(document); return skill; } /** * 基于 SKILL.md 内容和通用资源创建 Skill。 * - * @param id 仓储 ID,可为空 * @param skillContent SKILL.md 原始内容 * @param resources 通用资源列表 * @return Skill 聚合 */ - public static Skill createWithResources(String id, String skillContent, List resources) { - Skill skill = create(id, skillContent); + public static Skill createWithResources(String skillContent, List resources) { + Skill skill = create(skillContent); skill.setResources(resources); - SkillResources.refreshLegacyViews(skill); return skill; } - /** - * 基于 SKILL.md 和通用资源创建并严格校验正式标准 Skill。 - * - * @param id 仓储 ID,可为空 - * @param skillContent SKILL.md 原始内容 - * @param resources 通用资源列表 - * @return 通过正式标准校验的 Skill 聚合 - */ - public static Skill createWithResourcesStrict(String id, String skillContent, - List resources) { - Skill skill = createWithResources(id, skillContent, resources); - new DefaultSkillValidator().validate(skill); - return skill; - } - - /** - * 基于 SKILL.md 内容和资源列表创建 Skill。 - * - * @param id Skill ID - * @param skillContent SKILL.md 原始内容 - * @param references reference 文档列表 - * @param scripts script 脚本列表 - * @param assets asset 资产列表 - * @return Skill 聚合 - */ - public static Skill create(String id, String skillContent, List references, - List scripts, List assets) { - SkillDocument document = SkillFrontmatter.parseDocument(skillContent); - Map values = document.getFrontmatter().getValues(); - String name = requiredScalar(values, "name"); - String description = requiredScalar(values, "description"); - Skill skill = new Skill(); - skill.setId(id); - skill.setName(name); - skill.setDescription(description); - skill.setMetadata(new SkillMetadata(values)); - skill.setDocument(document); - skill.setReferences(references); - skill.setScripts(scripts); - skill.setAssets(assets); - skill.setResources(SkillResources.canonicalResources(skill)); - return skill; - } - - private static String requiredScalar(Map values, String key) { + private static void requiredScalar(Map values, String key) { Object value = values.get(key); if (!(value instanceof String text) || text.isBlank()) { throw new com.easyagents.skill.exception.SkillValidationException( "SKILL.md frontmatter " + key + " must be a non-blank string."); } - return text; } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java index 0cd7f9a..94145a3 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java @@ -1,7 +1,5 @@ package com.easyagents.skill.model; -import com.easyagents.skill.util.SkillResources; - import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -13,36 +11,10 @@ public class Skill implements Serializable { private static final long serialVersionUID = 1L; - private String id; private String packageRoot; - private String name; - private String description; - private SkillMetadata metadata = new SkillMetadata(); private String skillContent; private SkillDocument document; private List resources = new ArrayList<>(); - private boolean resourcesInitialized; - private List references = new ArrayList<>(); - private List scripts = new ArrayList<>(); - private List assets = new ArrayList<>(); - - /** - * 获取 Skill ID。 - * - * @return Skill ID - */ - public String getId() { - return id; - } - - /** - * 设置 Skill ID。 - * - * @param id Skill ID - */ - public void setId(String id) { - this.id = id; - } /** * 获取导入包中的顶层目录名;该值不是仓储 ID。 @@ -68,16 +40,7 @@ public class Skill implements Serializable { * @return 名称 */ public String getName() { - return name; - } - - /** - * 设置名称。 - * - * @param name 名称 - */ - public void setName(String name) { - this.name = name; + return frontmatterString("name"); } /** @@ -86,34 +49,7 @@ public class Skill implements Serializable { * @return 描述 */ public String getDescription() { - return description; - } - - /** - * 设置描述。 - * - * @param description 描述 - */ - public void setDescription(String description) { - this.description = description; - } - - /** - * 获取元数据。 - * - * @return 元数据 - */ - public SkillMetadata getMetadata() { - return metadata; - } - - /** - * 设置元数据。 - * - * @param metadata 元数据 - */ - public void setMetadata(SkillMetadata metadata) { - this.metadata = metadata == null ? new SkillMetadata() : metadata; + return frontmatterString("description"); } /** @@ -152,12 +88,6 @@ public class Skill implements Serializable { public void setDocument(SkillDocument document) { this.document = document; this.skillContent = document == null ? null : document.render(); - if (document != null) { - java.util.Map values = document.getFrontmatter().getValues(); - this.metadata = new SkillMetadata(values); - this.name = values.get("name") instanceof String text ? text : null; - this.description = values.get("description") instanceof String text ? text : null; - } } /** @@ -166,12 +96,6 @@ public class Skill implements Serializable { * @return 通用资源列表 */ public List getResources() { - if (!resourcesInitialized) { - resources = resources == null || resources.isEmpty() - ? new ArrayList<>(SkillResources.fromLegacyViews(this)) - : new ArrayList<>(resources); - resourcesInitialized = true; - } return resources; } @@ -182,81 +106,13 @@ public class Skill implements Serializable { */ public void setResources(List resources) { this.resources = resources == null ? new ArrayList<>() : new ArrayList<>(resources); - this.resourcesInitialized = true; } - /** - * 判断正式通用资源列表是否已被显式初始化。 - * - *

该标记用于区分“尚未迁移的旧资源视图”和“调用方明确设置的空资源列表”, - * 避免删除最后一个正式资源后又从旧兼容视图恢复该资源。

- * - * @return 已显式设置通用资源列表时为 true - */ - public boolean isResourcesInitialized() { - return resourcesInitialized; - } - - /** - * 获取参考文档列表。 - * - * @return 参考文档列表 - */ - public List getReferences() { - return references; - } - - /** - * 设置参考文档列表。 - * - * @param references 参考文档列表 - */ - public void setReferences(List references) { - this.references = references == null ? new ArrayList<>() : new ArrayList<>(references); - } - - /** - * 获取脚本列表。 - * - * @return 脚本列表 - */ - public List getScripts() { - return scripts; - } - - /** - * 设置脚本列表。 - * - * @param scripts 脚本列表 - */ - public void setScripts(List scripts) { - this.scripts = scripts == null ? new ArrayList<>() : new ArrayList<>(scripts); - } - - /** - * 获取资产列表。 - * - * @return 资产列表 - */ - public List getAssets() { - return assets; - } - - /** - * 设置资产列表。 - * - * @param assets 资产列表 - */ - public void setAssets(List assets) { - this.assets = assets == null ? new ArrayList<>() : new ArrayList<>(assets); - } - - /** - * 转换为轻量描述。 - * - * @return Skill 描述 - */ - public SkillDescriptor toDescriptor() { - return new SkillDescriptor(id, name, description, metadata); + private String frontmatterString(String key) { + if (document == null) { + return null; + } + Object value = document.getFrontmatter().get(key); + return value instanceof String text ? text : null; } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java deleted file mode 100644 index 796d7c7..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillAsset.java +++ /dev/null @@ -1,148 +0,0 @@ -package com.easyagents.skill.model; - -import java.io.Serializable; - -/** - * Skill 静态资产兼容视图。 - * - * @deprecated 请使用 {@link SkillResource}。 - */ -@Deprecated -public class SkillAsset implements Serializable { - - private static final long serialVersionUID = 1L; - - private String path; - private String name; - private String mediaType; - private String contentRef; - private String contentHash; - private long size; - private SkillMetadata metadata = new SkillMetadata(); - - /** - * 获取逻辑路径。 - * - * @return 逻辑路径 - */ - public String getPath() { - return path; - } - - /** - * 设置逻辑路径。 - * - * @param path 逻辑路径 - */ - public void setPath(String path) { - this.path = path; - } - - /** - * 获取名称。 - * - * @return 名称 - */ - public String getName() { - return name; - } - - /** - * 设置名称。 - * - * @param name 名称 - */ - public void setName(String name) { - this.name = name; - } - - /** - * 获取媒体类型。 - * - * @return 媒体类型 - */ - public String getMediaType() { - return mediaType; - } - - /** - * 设置媒体类型。 - * - * @param mediaType 媒体类型 - */ - public void setMediaType(String mediaType) { - this.mediaType = mediaType; - } - - /** - * 获取内容引用。 - * - * @return 内容引用 - */ - public String getContentRef() { - return contentRef; - } - - /** - * 设置内容引用。 - * - * @param contentRef 内容引用 - */ - public void setContentRef(String contentRef) { - this.contentRef = contentRef; - } - - /** - * 获取内容 hash。 - * - * @return 内容 hash - */ - public String getContentHash() { - return contentHash; - } - - /** - * 设置内容 hash。 - * - * @param contentHash 内容 hash - */ - public void setContentHash(String contentHash) { - this.contentHash = contentHash; - } - - /** - * 获取文件大小。 - * - * @return 文件大小 - */ - public long getSize() { - return size; - } - - /** - * 设置文件大小。 - * - * @param size 文件大小 - */ - public void setSize(long size) { - this.size = size; - } - - /** - * 获取元数据。 - * - * @return 元数据 - */ - public SkillMetadata getMetadata() { - return metadata; - } - - /** - * 设置元数据。 - * - * @param metadata 元数据 - */ - public void setMetadata(SkillMetadata metadata) { - this.metadata = metadata == null ? new SkillMetadata() : metadata; - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java deleted file mode 100644 index a25df10..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillDescriptor.java +++ /dev/null @@ -1,111 +0,0 @@ -package com.easyagents.skill.model; - -import java.io.Serial; -import java.io.Serializable; - -/** - * Skill 轻量描述。 - */ -public class SkillDescriptor implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; - - private String id; - private String name; - private String description; - private SkillMetadata metadata = new SkillMetadata(); - - /** - * 创建空 Skill 描述。 - */ - public SkillDescriptor() { - } - - /** - * 创建 Skill 描述。 - * - * @param id Skill ID - * @param name Skill 名称 - * @param description Skill 描述 - * @param metadata 元数据 - */ - public SkillDescriptor(String id, String name, String description, SkillMetadata metadata) { - this.id = id; - this.name = name; - this.description = description; - setMetadata(metadata); - } - - /** - * 获取 Skill ID。 - * - * @return Skill ID - */ - public String getId() { - return id; - } - - /** - * 设置 Skill ID。 - * - * @param id Skill ID - */ - public void setId(String id) { - this.id = id; - } - - /** - * 获取名称。 - * - * @return 名称 - */ - public String getName() { - return name; - } - - /** - * 设置名称。 - * - * @param name 名称 - */ - public void setName(String name) { - this.name = name; - } - - /** - * 获取描述。 - * - * @return 描述 - */ - public String getDescription() { - return description; - } - - /** - * 设置描述。 - * - * @param description 描述 - */ - public void setDescription(String description) { - this.description = description; - } - - /** - * 获取元数据。 - * - * @return 元数据 - */ - public SkillMetadata getMetadata() { - return metadata; - } - - /** - * 设置元数据。 - * - * @param metadata 元数据 - */ - public void setMetadata(SkillMetadata metadata) { - this.metadata = metadata == null ? new SkillMetadata() : metadata; - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java deleted file mode 100644 index 80657d1..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillReference.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.easyagents.skill.model; - -import java.io.Serializable; - -/** - * Skill Markdown 参考文档兼容视图。 - * - * @deprecated 请使用 {@link SkillResource}。 - */ -@Deprecated -public class SkillReference implements Serializable { - - private static final long serialVersionUID = 1L; - - private String path; - private String name; - private String content; - private String contentHash; - private long size; - private SkillMetadata metadata = new SkillMetadata(); - - /** - * 获取逻辑路径。 - * - * @return 逻辑路径 - */ - public String getPath() { - return path; - } - - /** - * 设置逻辑路径。 - * - * @param path 逻辑路径 - */ - public void setPath(String path) { - this.path = path; - } - - /** - * 获取名称。 - * - * @return 名称 - */ - public String getName() { - return name; - } - - /** - * 设置名称。 - * - * @param name 名称 - */ - public void setName(String name) { - this.name = name; - } - - /** - * 获取 Markdown 内容。 - * - * @return Markdown 内容 - */ - public String getContent() { - return content; - } - - /** - * 设置 Markdown 内容。 - * - * @param content Markdown 内容 - */ - public void setContent(String content) { - this.content = content; - } - - /** - * 获取内容 hash。 - * - * @return 内容 hash - */ - public String getContentHash() { - return contentHash; - } - - /** - * 设置内容 hash。 - * - * @param contentHash 内容 hash - */ - public void setContentHash(String contentHash) { - this.contentHash = contentHash; - } - - /** - * 获取文件大小。 - * - * @return 文件大小 - */ - public long getSize() { - return size; - } - - /** - * 设置文件大小。 - * - * @param size 文件大小 - */ - public void setSize(long size) { - this.size = size; - } - - /** - * 获取元数据。 - * - * @return 元数据 - */ - public SkillMetadata getMetadata() { - return metadata; - } - - /** - * 设置元数据。 - * - * @param metadata 元数据 - */ - public void setMetadata(SkillMetadata metadata) { - this.metadata = metadata == null ? new SkillMetadata() : metadata; - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java index cc950fe..9a7a5ae 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillResource.java @@ -18,7 +18,6 @@ public class SkillResource implements Serializable { private String contentRef; private String contentHash; private long size; - private SkillMetadata metadata = new SkillMetadata(); /** * 获取 Skill 根目录相对路径。 @@ -146,24 +145,6 @@ public class SkillResource implements Serializable { this.size = size; } - /** - * 获取资源扩展元数据。 - * - * @return 扩展元数据 - */ - public SkillMetadata getMetadata() { - return metadata; - } - - /** - * 设置资源扩展元数据。 - * - * @param metadata 扩展元数据 - */ - public void setMetadata(SkillMetadata metadata) { - this.metadata = metadata == null ? new SkillMetadata() : metadata; - } - /** * 判断资源是否以内联文本保存。 * diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java deleted file mode 100644 index 4679906..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScript.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.easyagents.skill.model; - -import java.io.Serializable; - -/** - * Skill 脚本源码兼容视图。 - * - * @deprecated 请使用 {@link SkillResource}。 - */ -@Deprecated -public class SkillScript implements Serializable { - - private static final long serialVersionUID = 1L; - - private String path; - private SkillScriptLanguage language = SkillScriptLanguage.UNKNOWN; - private String content; - private String contentHash; - private long size; - private SkillMetadata metadata = new SkillMetadata(); - - /** - * 获取逻辑路径。 - * - * @return 逻辑路径 - */ - public String getPath() { - return path; - } - - /** - * 设置逻辑路径。 - * - * @param path 逻辑路径 - */ - public void setPath(String path) { - this.path = path; - } - - /** - * 获取脚本语言。 - * - * @return 脚本语言 - */ - public SkillScriptLanguage getLanguage() { - return language; - } - - /** - * 设置脚本语言。 - * - * @param language 脚本语言 - */ - public void setLanguage(SkillScriptLanguage language) { - this.language = language == null ? SkillScriptLanguage.UNKNOWN : language; - } - - /** - * 获取脚本源码。 - * - * @return 脚本源码 - */ - public String getContent() { - return content; - } - - /** - * 设置脚本源码。 - * - * @param content 脚本源码 - */ - public void setContent(String content) { - this.content = content; - } - - /** - * 获取内容 hash。 - * - * @return 内容 hash - */ - public String getContentHash() { - return contentHash; - } - - /** - * 设置内容 hash。 - * - * @param contentHash 内容 hash - */ - public void setContentHash(String contentHash) { - this.contentHash = contentHash; - } - - /** - * 获取文件大小。 - * - * @return 文件大小 - */ - public long getSize() { - return size; - } - - /** - * 设置文件大小。 - * - * @param size 文件大小 - */ - public void setSize(long size) { - this.size = size; - } - - /** - * 获取元数据。 - * - * @return 元数据 - */ - public SkillMetadata getMetadata() { - return metadata; - } - - /** - * 设置元数据。 - * - * @param metadata 元数据 - */ - public void setMetadata(SkillMetadata metadata) { - this.metadata = metadata == null ? new SkillMetadata() : metadata; - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java deleted file mode 100644 index c9ce329..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillScriptLanguage.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.easyagents.skill.model; - -/** - * Skill 脚本语言。 - */ -public enum SkillScriptLanguage { - - /** - * Python 脚本。 - */ - PYTHON, - - /** - * JavaScript 脚本。 - */ - JAVASCRIPT, - - /** - * Shell 脚本。 - */ - SHELL, - - /** - * 未知脚本语言。 - */ - UNKNOWN; - - /** - * 按脚本路径识别语言。 - * - * @param path 脚本逻辑路径 - * @return 脚本语言 - */ - public static SkillScriptLanguage fromPath(String path) { - if (path == null) { - return UNKNOWN; - } - String lower = path.toLowerCase(); - if (lower.endsWith(".py")) { - return PYTHON; - } - if (lower.endsWith(".js")) { - return JAVASCRIPT; - } - if (lower.endsWith(".sh")) { - return SHELL; - } - return UNKNOWN; - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java deleted file mode 100644 index 03552ee..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/SkillRepository.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.easyagents.skill.repository; - -import com.easyagents.skill.model.Skill; -import com.easyagents.skill.model.SkillDescriptor; - -import java.util.List; -import java.util.Optional; - -/** - * Skill 聚合存储接口。 - */ -public interface SkillRepository { - - /** - * 保存 Skill。 - * - * @param skill Skill 聚合 - */ - void save(Skill skill); - - /** - * 获取完整 Skill。 - * - * @param skillId Skill ID - * @return Skill 聚合 - */ - Optional get(String skillId); - - /** - * 获取 Skill 描述。 - * - * @param skillId Skill ID - * @return Skill 描述 - */ - Optional getDescriptor(String skillId); - - /** - * 列出 Skill 描述。 - * - * @return Skill 描述列表 - */ - List listDescriptors(); - - /** - * 删除 Skill。 - * - * @param skillId Skill ID - */ - void delete(String skillId); - - /** - * 判断 Skill 是否存在。 - * - * @param skillId Skill ID - * @return 存在时为 true - */ - boolean exists(String skillId); -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java b/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java deleted file mode 100644 index 867e1b4..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/repository/memory/InMemorySkillRepository.java +++ /dev/null @@ -1,214 +0,0 @@ -package com.easyagents.skill.repository.memory; - -import com.easyagents.skill.exception.SkillValidationException; -import com.easyagents.skill.model.*; -import com.easyagents.skill.repository.SkillRepository; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * 基于内存的 Skill 聚合仓储实现。 - */ -public class InMemorySkillRepository implements SkillRepository { - - private final ConcurrentMap skills = new ConcurrentHashMap<>(); - - /** - * 保存 Skill 聚合。 - * - * @param skill Skill 聚合 - */ - @Override - public void save(Skill skill) { - if (skill == null || isBlank(skill.getId())) { - throw new SkillValidationException("Skill id is required."); - } - skills.put(skill.getId(), copySkill(skill)); - } - - /** - * 获取完整 Skill。 - * - * @param skillId Skill ID - * @return Skill 聚合 - */ - @Override - public Optional get(String skillId) { - Skill skill = skills.get(skillId); - return skill == null ? Optional.empty() : Optional.of(copySkill(skill)); - } - - /** - * 获取 Skill 描述。 - * - * @param skillId Skill ID - * @return Skill 描述 - */ - @Override - public Optional getDescriptor(String skillId) { - Skill skill = skills.get(skillId); - return skill == null ? Optional.empty() : Optional.of(copyDescriptor(skill.toDescriptor())); - } - - /** - * 列出 Skill 描述。 - * - * @return Skill 描述列表 - */ - @Override - public List listDescriptors() { - List descriptors = new ArrayList<>(); - for (Skill skill : skills.values()) { - descriptors.add(copyDescriptor(skill.toDescriptor())); - } - return descriptors; - } - - /** - * 删除 Skill。 - * - * @param skillId Skill ID - */ - @Override - public void delete(String skillId) { - skills.remove(skillId); - } - - /** - * 判断 Skill 是否存在。 - * - * @param skillId Skill ID - * @return 存在时为 true - */ - @Override - public boolean exists(String skillId) { - return skills.containsKey(skillId); - } - - private static Skill copySkill(Skill source) { - Skill target = new Skill(); - target.setId(source.getId()); - target.setPackageRoot(source.getPackageRoot()); - target.setName(source.getName()); - target.setDescription(source.getDescription()); - target.setMetadata(copyMetadata(source.getMetadata())); - SkillDocument copiedDocument = copyDocument(source.getDocument()); - if (copiedDocument == null) { - target.setSkillContent(source.getSkillContent()); - } else { - target.setDocument(copiedDocument); - } - target.setResources(copyResources(source.getResources())); - target.setReferences(copyReferences(source.getReferences())); - target.setScripts(copyScripts(source.getScripts())); - target.setAssets(copyAssets(source.getAssets())); - return target; - } - - private static SkillDocument copyDocument(SkillDocument source) { - if (source == null) { - return null; - } - SkillDocument target = new SkillDocument(source.getRawContent(), - source.getFrontmatter().getValues(), source.getMarkdownBody(), - source.getFrontmatterLocations(), source.getDiagnostics()); - target.setModified(source.isModified()); - return target; - } - - private static List copyResources(List sources) { - List targets = new ArrayList<>(); - if (sources == null) { - return targets; - } - for (SkillResource source : sources) { - SkillResource target = new SkillResource(); - target.setPath(source.getPath()); - target.setKind(source.getKind()); - target.setMediaType(source.getMediaType()); - target.setTextContent(source.getTextContent()); - target.setContentRef(source.getContentRef()); - target.setContentHash(source.getContentHash()); - target.setSize(source.getSize()); - target.setMetadata(copyMetadata(source.getMetadata())); - targets.add(target); - } - return targets; - } - - private static List copyReferences(List sources) { - List targets = new ArrayList<>(); - if (sources == null) { - return targets; - } - for (SkillReference source : sources) { - SkillReference target = new SkillReference(); - target.setPath(source.getPath()); - target.setName(source.getName()); - target.setContent(source.getContent()); - target.setContentHash(source.getContentHash()); - target.setSize(source.getSize()); - target.setMetadata(copyMetadata(source.getMetadata())); - targets.add(target); - } - return targets; - } - - private static List copyScripts(List sources) { - List targets = new ArrayList<>(); - if (sources == null) { - return targets; - } - for (SkillScript source : sources) { - SkillScript target = new SkillScript(); - target.setPath(source.getPath()); - target.setLanguage(source.getLanguage()); - target.setContent(source.getContent()); - target.setContentHash(source.getContentHash()); - target.setSize(source.getSize()); - target.setMetadata(copyMetadata(source.getMetadata())); - targets.add(target); - } - return targets; - } - - private static List copyAssets(List sources) { - List targets = new ArrayList<>(); - if (sources == null) { - return targets; - } - for (SkillAsset source : sources) { - SkillAsset target = new SkillAsset(); - target.setPath(source.getPath()); - target.setName(source.getName()); - target.setMediaType(source.getMediaType()); - target.setContentRef(source.getContentRef()); - target.setContentHash(source.getContentHash()); - target.setSize(source.getSize()); - target.setMetadata(copyMetadata(source.getMetadata())); - targets.add(target); - } - return targets; - } - - private static SkillDescriptor copyDescriptor(SkillDescriptor source) { - return new SkillDescriptor( - source.getId(), - source.getName(), - source.getDescription(), - copyMetadata(source.getMetadata()) - ); - } - - private static SkillMetadata copyMetadata(SkillMetadata source) { - return source == null ? new SkillMetadata() : new SkillMetadata(source.getValues()); - } - - private static boolean isBlank(String value) { - return value == null || value.isBlank(); - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java index c94e138..6db9b80 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStage.java @@ -1,6 +1,5 @@ package com.easyagents.skill.store; -import java.nio.file.Path; import java.util.Objects; /** @@ -12,8 +11,6 @@ public final class SkillContentStage { private final String contentRef; private final String contentHash; private final long size; - private final boolean alreadyCommitted; - private final Path compatibilityPath; /** * 创建内容阶段结果。 @@ -22,21 +19,12 @@ public final class SkillContentStage { * @param contentRef 最终内容引用 * @param contentHash SHA-256 * @param size 字节大小 - * @param alreadyCommitted 是否由兼容实现提前写入正式存储 */ - public SkillContentStage(String stageId, String contentRef, String contentHash, - long size, boolean alreadyCommitted) { - this(stageId, contentRef, contentHash, size, alreadyCommitted, null); - } - - private SkillContentStage(String stageId, String contentRef, String contentHash, - long size, boolean alreadyCommitted, Path compatibilityPath) { + public SkillContentStage(String stageId, String contentRef, String contentHash, long size) { this.stageId = Objects.requireNonNull(stageId, "stageId"); this.contentRef = Objects.requireNonNull(contentRef, "contentRef"); this.contentHash = Objects.requireNonNull(contentHash, "contentHash"); this.size = size; - this.alreadyCommitted = alreadyCommitted; - this.compatibilityPath = compatibilityPath; } /** @return 暂存标识 */ @@ -59,18 +47,4 @@ public final class SkillContentStage { return size; } - /** @return 已提前提交时为 true */ - public boolean isAlreadyCommitted() { - return alreadyCommitted; - } - - static SkillContentStage compatibility(Path path, String contentHash, long size) { - String contentRef = "sha256:" + contentHash; - return new SkillContentStage(contentRef, contentRef, contentHash, - size, false, path); - } - - Path compatibilityPath() { - return compatibilityPath; - } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java index 99f39af..404f094 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/SkillContentStore.java @@ -1,16 +1,6 @@ package com.easyagents.skill.store; -import com.easyagents.skill.exception.SkillException; -import com.easyagents.skill.util.SkillHashes; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.security.DigestOutputStream; -import java.security.MessageDigest; /** * Skill 二进制内容流式存储与引用生命周期接口。 @@ -25,62 +15,14 @@ public interface SkillContentStore { */ String put(byte[] bytes); - /** - * 流式保存内容并返回内容引用。 - * - *

兼容默认实现仅缓存单个文件;正式持久化实现应覆盖该方法以直接流式写入。

- * - * @param inputStream 内容流,不由本方法关闭 - * @param maxBytes 最大允许字节数 - * @return 内容引用 - */ - default String put(InputStream inputStream, long maxBytes) { - return put(readBounded(inputStream, maxBytes)); - } - /** * 暂存内容,供完成全包校验后统一提交。 * - *

为兼容旧实现,默认实现会立即写入;正式实现应覆盖并提供真实暂存区。

- * * @param inputStream 内容流,不由本方法关闭 * @param maxBytes 最大允许字节数 * @return 暂存结果 */ - default SkillContentStage stage(InputStream inputStream, long maxBytes) { - if (inputStream == null || maxBytes < 0) { - throw new SkillException("Valid Skill content stream and limit are required."); - } - Path temporaryFile = null; - try { - temporaryFile = Files.createTempFile("easy-agents-skill-content-", ".stage"); - MessageDigest digest = SkillHashes.newSha256Digest(); - long total = 0; - try (DigestOutputStream output = new DigestOutputStream( - Files.newOutputStream(temporaryFile, StandardOpenOption.TRUNCATE_EXISTING), digest)) { - byte[] buffer = new byte[16 * 1024]; - int read; - while ((read = inputStream.read(buffer)) >= 0) { - if (read == 0) { - continue; - } - if (total > maxBytes - read) { - throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); - } - output.write(buffer, 0, read); - total += read; - } - } - return SkillContentStage.compatibility( - temporaryFile, SkillHashes.toHex(digest.digest()), total); - } catch (IOException | RuntimeException e) { - deleteTemporaryFile(temporaryFile); - if (e instanceof SkillException skillException) { - throw skillException; - } - throw new SkillException("Failed to stage Skill content stream.", e); - } - } + SkillContentStage stage(InputStream inputStream, long maxBytes); /** * 提交暂存内容。 @@ -88,59 +30,28 @@ public interface SkillContentStore { * @param stage 暂存结果 * @return 最终内容引用 */ - default String commit(SkillContentStage stage) { - if (stage == null) { - throw new SkillException("Skill content stage is required."); - } - Path compatibilityPath = stage.compatibilityPath(); - if (compatibilityPath != null) { - try (InputStream input = Files.newInputStream(compatibilityPath)) { - String contentRef = put(input, stage.getSize()); - if (!stage.getContentRef().equals(contentRef)) { - release(contentRef); - throw new SkillException("Skill content store returned a non-hash content reference."); - } - return contentRef; - } catch (IOException e) { - throw new SkillException("Failed to commit staged Skill content.", e); - } finally { - deleteTemporaryFile(compatibilityPath); - } - } - return stage.getContentRef(); - } + String commit(SkillContentStage stage); /** * 回滚尚未提交的内容。 * * @param stage 暂存结果 */ - default void rollback(SkillContentStage stage) { - if (stage != null) { - deleteTemporaryFile(stage.compatibilityPath()); - if (stage.isAlreadyCommitted()) { - release(stage.getContentRef()); - } - } - } + void rollback(SkillContentStage stage); /** * 增加正式内容引用计数。 * * @param contentRef 内容引用 */ - default void retain(String contentRef) { - // 旧实现没有引用计数,保留兼容空操作。 - } + void retain(String contentRef); /** * 释放正式内容引用;引用归零后实现可以删除物理内容。 * * @param contentRef 内容引用 */ - default void release(String contentRef) { - // 旧实现没有引用计数,保留兼容空操作。 - } + void release(String contentRef); /** * 打开内容流。 @@ -150,14 +61,6 @@ public interface SkillContentStore { */ InputStream open(String contentRef); - /** - * 读取全部内容。 - * - * @param contentRef 内容引用 - * @return 内容字节 - */ - byte[] readAllBytes(String contentRef); - /** * 判断内容是否存在。 * @@ -165,43 +68,4 @@ public interface SkillContentStore { * @return 存在时为 true */ boolean exists(String contentRef); - - private static byte[] readBounded(InputStream inputStream, long maxBytes) { - if (inputStream == null) { - throw new SkillException("Skill content input stream is required."); - } - if (maxBytes < 0) { - throw new SkillException("Skill content max bytes cannot be negative."); - } - try { - ByteArrayOutputStream output = new ByteArrayOutputStream((int) Math.min(maxBytes, 8_192)); - byte[] buffer = new byte[8_192]; - long total = 0; - int read; - while ((read = inputStream.read(buffer)) >= 0) { - if (read == 0) { - continue; - } - total += read; - if (total > maxBytes) { - throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); - } - output.write(buffer, 0, read); - } - return output.toByteArray(); - } catch (IOException e) { - throw new SkillException("Failed to read Skill content stream.", e); - } - } - - private static void deleteTemporaryFile(Path path) { - if (path == null) { - return; - } - try { - Files.deleteIfExists(path); - } catch (IOException ignored) { - // 临时文件清理由操作系统兜底,调用方异常语义保持不变。 - } - } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java index 7a39bdd..7302814 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStore.java @@ -86,19 +86,7 @@ public final class TemporaryFileSkillContentStore implements SkillContentStore, @Override public String put(byte[] bytes) { byte[] safeBytes = bytes == null ? new byte[0] : bytes; - return put(new ByteArrayInputStream(safeBytes), safeBytes.length); - } - - /** - * 流式保存内容并持有一个正式引用。 - * - * @param inputStream 内容流,不由本方法关闭 - * @param maxBytes 最大允许字节数 - * @return SHA-256 内容引用 - */ - @Override - public String put(InputStream inputStream, long maxBytes) { - SkillContentStage stage = stage(inputStream, maxBytes); + SkillContentStage stage = stage(new ByteArrayInputStream(safeBytes), safeBytes.length); try { return commit(stage); } catch (RuntimeException exception) { @@ -153,7 +141,7 @@ public final class TemporaryFileSkillContentStore implements SkillContentStore, ensureOpenLocked(); stagedContents.put(stageId, stagedContent); } - return new SkillContentStage(stageId, contentRef, contentHash, size, false); + return new SkillContentStage(stageId, contentRef, contentHash, size); } catch (IOException | RuntimeException exception) { deleteFileQuietly(stagedPath); if (exception instanceof SkillException skillException) { @@ -309,24 +297,6 @@ public final class TemporaryFileSkillContentStore implements SkillContentStore, } } - /** - * 读取正式内容的全部字节。 - * - *

该兼容方法会按接口约定返回一个字节数组;流式调用方应优先使用 {@link #open(String)}。

- * - * @param contentRef 内容引用 - * @return 内容字节 - */ - @Override - public byte[] readAllBytes(String contentRef) { - try (InputStream input = open(contentRef)) { - return input.readAllBytes(); - } catch (IOException exception) { - throw new SkillException("Failed to read temporary Skill content: " + contentRef, - exception); - } - } - /** * 判断正式内容是否仍有有效引用。 * diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java b/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java deleted file mode 100644 index e33f711..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java +++ /dev/null @@ -1,210 +0,0 @@ -package com.easyagents.skill.store.memory; - -import com.easyagents.skill.exception.SkillException; -import com.easyagents.skill.store.SkillContentStage; -import com.easyagents.skill.store.SkillContentStore; -import com.easyagents.skill.util.SkillHashes; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * 基于内存的 Skill 内容存储,支持真实暂存与引用计数,适用于测试和轻量场景。 - */ -public class InMemorySkillContentStore implements SkillContentStore { - - private final ConcurrentMap contents = new ConcurrentHashMap<>(); - private final ConcurrentMap stagedContents = new ConcurrentHashMap<>(); - - /** - * 保存内容并持有一个引用。 - * - * @param bytes 内容字节 - * @return 内容引用 - */ - @Override - public String put(byte[] bytes) { - byte[] safeBytes = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); - String contentRef = SkillHashes.sha256Ref(safeBytes); - contents.compute(contentRef, (key, current) -> { - if (current == null) { - return new StoredContent(safeBytes); - } - if (!Arrays.equals(current.bytes, safeBytes)) { - throw new SkillException("SHA-256 collision detected for Skill content: " + contentRef); - } - current.references.incrementAndGet(); - return current; - }); - return contentRef; - } - - /** - * 流式保存内容并持有一个引用。 - * - * @param inputStream 内容流 - * @param maxBytes 最大允许字节数 - * @return 内容引用 - */ - @Override - public String put(InputStream inputStream, long maxBytes) { - return put(readBounded(inputStream, maxBytes)); - } - - /** - * 将内容写入独立暂存区。 - * - * @param inputStream 内容流 - * @param maxBytes 最大允许字节数 - * @return 暂存结果 - */ - @Override - public SkillContentStage stage(InputStream inputStream, long maxBytes) { - byte[] bytes = readBounded(inputStream, maxBytes); - String hash = SkillHashes.sha256Hex(bytes); - String stageId = UUID.randomUUID().toString(); - stagedContents.put(stageId, bytes); - return new SkillContentStage(stageId, "sha256:" + hash, hash, bytes.length, false); - } - - /** - * 原子地将暂存内容转为正式引用。 - * - * @param stage 暂存结果 - * @return 正式内容引用 - */ - @Override - public String commit(SkillContentStage stage) { - if (stage == null) { - throw new SkillException("Skill content stage is required."); - } - byte[] bytes = stagedContents.remove(stage.getStageId()); - if (bytes == null) { - throw new SkillException("Skill content stage does not exist: " + stage.getStageId()); - } - String contentRef = put(bytes); - if (!contentRef.equals(stage.getContentRef())) { - release(contentRef); - throw new SkillException("Skill staged content hash changed before commit."); - } - return contentRef; - } - - /** - * 删除暂存内容。 - * - * @param stage 暂存结果 - */ - @Override - public void rollback(SkillContentStage stage) { - if (stage != null) { - stagedContents.remove(stage.getStageId()); - } - } - - /** - * 增加正式内容引用计数。 - * - * @param contentRef 内容引用 - */ - @Override - public void retain(String contentRef) { - contents.compute(contentRef, (key, content) -> { - if (content == null) { - throw new SkillException("Skill content does not exist: " + contentRef); - } - content.references.incrementAndGet(); - return content; - }); - } - - /** - * 释放正式内容引用并在归零时删除内容。 - * - * @param contentRef 内容引用 - */ - @Override - public void release(String contentRef) { - contents.computeIfPresent(contentRef, (key, content) -> - content.references.decrementAndGet() <= 0 ? null : content); - } - - /** - * 打开内容流。 - * - * @param contentRef 内容引用 - * @return 内容流 - */ - @Override - public InputStream open(String contentRef) { - return new ByteArrayInputStream(readAllBytes(contentRef)); - } - - /** - * 读取全部内容。 - * - * @param contentRef 内容引用 - * @return 内容副本 - */ - @Override - public byte[] readAllBytes(String contentRef) { - StoredContent content = contents.get(contentRef); - if (content == null) { - throw new SkillException("Skill content does not exist: " + contentRef); - } - return Arrays.copyOf(content.bytes, content.bytes.length); - } - - /** - * 判断内容是否存在。 - * - * @param contentRef 内容引用 - * @return 存在时为 true - */ - @Override - public boolean exists(String contentRef) { - return contents.containsKey(contentRef); - } - - private static byte[] readBounded(InputStream inputStream, long maxBytes) { - if (inputStream == null || maxBytes < 0) { - throw new SkillException("Valid Skill content stream and limit are required."); - } - try { - ByteArrayOutputStream output = new ByteArrayOutputStream((int) Math.min(maxBytes, 8_192)); - byte[] buffer = new byte[8_192]; - long total = 0; - int read; - while ((read = inputStream.read(buffer)) >= 0) { - if (read == 0) { - continue; - } - total += read; - if (total > maxBytes) { - throw new SkillException("Skill content exceeds " + maxBytes + " bytes."); - } - output.write(buffer, 0, read); - } - return output.toByteArray(); - } catch (IOException e) { - throw new SkillException("Failed to read Skill content stream.", e); - } - } - - private static final class StoredContent { - - private final byte[] bytes; - private final AtomicInteger references = new AtomicInteger(1); - - private StoredContent(byte[] bytes) { - this.bytes = Arrays.copyOf(bytes, bytes.length); - } - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java index 67f9ed9..48bb64d 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillResources.java @@ -1,29 +1,25 @@ package com.easyagents.skill.util; -import com.easyagents.skill.model.Skill; -import com.easyagents.skill.model.SkillAsset; -import com.easyagents.skill.model.SkillMetadata; -import com.easyagents.skill.model.SkillReference; -import com.easyagents.skill.model.SkillResource; import com.easyagents.skill.model.SkillResourceKind; -import com.easyagents.skill.model.SkillScript; -import com.easyagents.skill.model.SkillScriptLanguage; -import java.util.ArrayList; -import java.util.List; import java.util.Locale; import java.util.Set; /** - * 通用 Skill 资源与旧资源视图之间的兼容适配工具。 + * 通用 Skill 资源识别工具。 */ public final class SkillResources { private static final Set TEXT_EXTENSIONS = Set.of( ".md", ".markdown", ".txt", ".json", ".yaml", ".yml", ".xml", ".csv", ".tsv", - ".htm", ".html", ".css", ".properties", ".toml", ".ini", ".sql", ".java", ".kt", ".kts", + ".htm", ".html", ".svg", ".css", ".scss", ".less", ".properties", ".toml", ".ini", + ".conf", ".config", ".env", ".sql", ".graphql", ".gql", ".proto", ".java", ".kt", ".kts", ".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".sh", ".bash", ".zsh", - ".rb", ".go", ".rs", ".c", ".h", ".cpp", ".hpp", ".gradle", ".groovy" + ".rb", ".go", ".rs", ".c", ".h", ".cpp", ".hpp", ".gradle", ".groovy", ".vue", ".svelte" + ); + + private static final Set TEXT_FILE_NAMES = Set.of( + "dockerfile", "makefile", "gradlew", ".gitignore", ".gitattributes", ".editorconfig" ); private SkillResources() { @@ -50,138 +46,22 @@ public final class SkillResources { * 判断资源是否应按严格 UTF-8 文本处理。 * * @param path 资源路径 - * @param kind 资源语义类型 * @param mediaType 媒体类型 * @return 文本资源时为 true */ - public static boolean isText(String path, SkillResourceKind kind, String mediaType) { - if (kind == SkillResourceKind.SCRIPT) { - return true; - } - if (kind == SkillResourceKind.ASSET) { - return false; - } - if (mediaType != null && (mediaType.startsWith("text/") - || mediaType.contains("json") || mediaType.contains("yaml") - || mediaType.contains("xml") || mediaType.contains("javascript"))) { + public static boolean isText(String path, String mediaType) { + String normalizedMediaType = mediaType == null ? "" : mediaType.toLowerCase(Locale.ROOT); + if (normalizedMediaType.startsWith("text/") + || normalizedMediaType.contains("json") || normalizedMediaType.contains("yaml") + || normalizedMediaType.contains("xml") || normalizedMediaType.contains("javascript") + || normalizedMediaType.contains("typescript") || normalizedMediaType.contains("toml") + || normalizedMediaType.contains("sql")) { return true; } String lowerPath = path.toLowerCase(Locale.ROOT); - return TEXT_EXTENSIONS.stream().anyMatch(lowerPath::endsWith); + String fileName = SkillPaths.fileName(lowerPath); + return TEXT_FILE_NAMES.contains(fileName) + || TEXT_EXTENSIONS.stream().anyMatch(lowerPath::endsWith); } - /** - * 获取 Skill 的正式通用资源;旧模型会被按需转换。 - * - * @param skill Skill 聚合 - * @return 通用资源副本 - */ - public static List canonicalResources(Skill skill) { - if (skill == null) { - return new ArrayList<>(); - } - return new ArrayList<>(skill.getResources()); - } - - /** - * 将尚未迁移的旧 references、scripts、assets 视图转换为正式通用资源。 - * - *

该方法只读取旧视图,不读取或修改正式资源列表,由 {@link Skill#getResources()} - * 在第一次正式访问时完成一次性迁移。

- * - * @param skill Skill 聚合 - * @return 从旧视图转换得到的通用资源副本 - */ - public static List fromLegacyViews(Skill skill) { - List resources = new ArrayList<>(); - if (skill == null) { - return resources; - } - if (skill.getReferences() != null) { - for (SkillReference reference : skill.getReferences()) { - SkillResource resource = base(reference.getPath(), SkillResourceKind.REFERENCE, - "text/markdown", reference.getContentHash(), reference.getSize(), reference.getMetadata()); - resource.setTextContent(reference.getContent()); - resources.add(resource); - } - } - if (skill.getScripts() != null) { - for (SkillScript script : skill.getScripts()) { - SkillResource resource = base(script.getPath(), SkillResourceKind.SCRIPT, - "text/plain", script.getContentHash(), script.getSize(), script.getMetadata()); - resource.setTextContent(script.getContent()); - resources.add(resource); - } - } - if (skill.getAssets() != null) { - for (SkillAsset asset : skill.getAssets()) { - SkillResource resource = base(asset.getPath(), SkillResourceKind.ASSET, - asset.getMediaType(), asset.getContentHash(), asset.getSize(), asset.getMetadata()); - resource.setContentRef(asset.getContentRef()); - resources.add(resource); - } - } - return resources; - } - - /** - * 依据通用资源刷新旧 references、scripts、assets 兼容视图。 - * - * @param skill Skill 聚合 - */ - public static void refreshLegacyViews(Skill skill) { - List references = new ArrayList<>(); - List scripts = new ArrayList<>(); - List assets = new ArrayList<>(); - for (SkillResource resource : skill.getResources()) { - if (resource.getKind() == SkillResourceKind.REFERENCE && resource.isText()) { - SkillReference reference = new SkillReference(); - reference.setPath(resource.getPath()); - reference.setName(SkillPaths.fileName(resource.getPath())); - reference.setContent(resource.getTextContent()); - reference.setContentHash(resource.getContentHash()); - reference.setSize(resource.getSize()); - reference.setMetadata(copy(resource.getMetadata())); - references.add(reference); - } else if (resource.getKind() == SkillResourceKind.SCRIPT && resource.isText()) { - SkillScript script = new SkillScript(); - script.setPath(resource.getPath()); - script.setLanguage(SkillScriptLanguage.fromPath(resource.getPath())); - script.setContent(resource.getTextContent()); - script.setContentHash(resource.getContentHash()); - script.setSize(resource.getSize()); - script.setMetadata(copy(resource.getMetadata())); - scripts.add(script); - } else if (resource.getKind() == SkillResourceKind.ASSET) { - SkillAsset asset = new SkillAsset(); - asset.setPath(resource.getPath()); - asset.setName(SkillPaths.fileName(resource.getPath())); - asset.setMediaType(resource.getMediaType()); - asset.setContentRef(resource.getContentRef()); - asset.setContentHash(resource.getContentHash()); - asset.setSize(resource.getSize()); - asset.setMetadata(copy(resource.getMetadata())); - assets.add(asset); - } - } - skill.setReferences(references); - skill.setScripts(scripts); - skill.setAssets(assets); - } - - private static SkillResource base(String path, SkillResourceKind kind, String mediaType, - String hash, long size, SkillMetadata metadata) { - SkillResource resource = new SkillResource(); - resource.setPath(path); - resource.setKind(kind); - resource.setMediaType(mediaType); - resource.setContentHash(hash); - resource.setSize(size); - resource.setMetadata(copy(metadata)); - return resource; - } - - private static SkillMetadata copy(SkillMetadata metadata) { - return metadata == null ? new SkillMetadata() : new SkillMetadata(metadata.getValues()); - } } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java deleted file mode 100644 index 8654b40..0000000 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/SkillValidator.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.easyagents.skill.validation; - -import com.easyagents.skill.exception.SkillValidationException; -import com.easyagents.skill.model.Skill; -import com.easyagents.skill.model.SkillPackageLimits; - -/** - * Skill 校验接口。 - */ -public interface SkillValidator { - - /** - * 校验 Skill 聚合。 - * - * @param skill Skill 聚合 - */ - void validate(Skill skill); - - /** - * 聚合校验 Skill 并返回结构化报告。 - * - *

兼容默认实现会把旧命令式异常转换为单个错误;新实现应覆盖以返回全部问题。

- * - * @param skill Skill 聚合 - * @return 结构化校验报告 - */ - default SkillValidationReport validateReport(Skill skill) { - SkillValidationReport report = new SkillValidationReport(); - try { - validate(skill); - } catch (SkillValidationException e) { - report.add(new SkillValidationIssue(e.getCode(), SkillValidationSeverity.ERROR, e.getPath(), - e.getLine(), e.getColumn(), e.getMessage(), null)); - } - return report; - } - - /** - * 使用当前 Codec 操作的安全限额聚合校验 Skill。 - * - *

兼容实现默认委托给原有校验入口;需要检查包限额的实现应覆盖本方法。

- * - * @param skill Skill 聚合 - * @param limits 当前读写操作的安全限额 - * @return 结构化校验报告 - */ - default SkillValidationReport validateReport(Skill skill, SkillPackageLimits limits) { - return validateReport(skill); - } - - /** - * 使用指定标准模式和当前 Codec 安全限额聚合校验 Skill。 - * - *

兼容实现默认忽略模式;需要区分草稿导入与正式标准约束的实现应覆盖本方法。

- * - * @param skill Skill 聚合 - * @param limits 当前读写操作的安全限额 - * @param mode 标准校验模式 - * @return 结构化校验报告 - */ - default SkillValidationReport validateReport(Skill skill, SkillPackageLimits limits, - SkillValidationMode mode) { - return validateReport(skill, limits); - } -} diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java index 7cf7ba7..42e7366 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/validation/defaults/DefaultSkillValidator.java @@ -6,7 +6,6 @@ import com.easyagents.skill.model.SkillDocument; import com.easyagents.skill.model.SkillPackageLimits; import com.easyagents.skill.model.SkillResource; import com.easyagents.skill.model.SkillResourceKind; -import com.easyagents.skill.model.SkillScriptLanguage; import com.easyagents.skill.model.SkillSourceLocation; import com.easyagents.skill.util.SkillFrontmatter; import com.easyagents.skill.util.SkillHashes; @@ -17,7 +16,6 @@ import com.easyagents.skill.validation.SkillValidationIssue; import com.easyagents.skill.validation.SkillValidationMode; import com.easyagents.skill.validation.SkillValidationReport; import com.easyagents.skill.validation.SkillValidationSeverity; -import com.easyagents.skill.validation.SkillValidator; import java.nio.charset.CharacterCodingException; import java.util.HashMap; @@ -31,7 +29,7 @@ import java.util.regex.Pattern; /** * 默认 Skill 聚合结构化校验器。 */ -public class DefaultSkillValidator implements SkillValidator { +public class DefaultSkillValidator { private static final Pattern CANONICAL_NAME = Pattern.compile("[a-z0-9]+(?:-[a-z0-9]+)*"); private static final Pattern LEGACY_UNDERSCORE_NAME = Pattern.compile("[a-z0-9]+(?:_[a-z0-9]+)+"); @@ -61,7 +59,6 @@ public class DefaultSkillValidator implements SkillValidator { * @param skill Skill 聚合 * @throws SkillValidationException 校验失败 */ - @Override public void validate(Skill skill) { validateReport(skill, limits, SkillValidationMode.STANDARD).throwIfInvalid(); } @@ -72,7 +69,6 @@ public class DefaultSkillValidator implements SkillValidator { * @param skill Skill 聚合 * @return 结构化校验报告 */ - @Override public SkillValidationReport validateReport(Skill skill) { return validateReport(skill, limits); } @@ -84,7 +80,6 @@ public class DefaultSkillValidator implements SkillValidator { * @param operationLimits 当前读写操作的安全限额 * @return 结构化校验报告 */ - @Override public SkillValidationReport validateReport(Skill skill, SkillPackageLimits operationLimits) { return validateReport(skill, operationLimits, SkillValidationMode.DRAFT_IMPORT); } @@ -97,7 +92,6 @@ public class DefaultSkillValidator implements SkillValidator { * @param mode 标准校验模式 * @return 结构化校验报告 */ - @Override public SkillValidationReport validateReport(Skill skill, SkillPackageLimits operationLimits, SkillValidationMode mode) { SkillPackageLimits effectiveLimits = operationLimits == null ? limits : operationLimits; @@ -110,11 +104,11 @@ public class DefaultSkillValidator implements SkillValidator { SkillDocument document = parseDocument(skill, report, effectiveLimits); validateName(skill, document, report, effectiveMode); - validateDescription(skill, document, report); + validateDescription(document, report); if (document != null) { - validateDocument(skill, document, report); + validateDocument(document, report); } - List resources = SkillResources.canonicalResources(skill); + List resources = skill.getResources(); validateAggregateLimits(skill, resources, report, effectiveLimits); validateResources(skill.getName(), resources, report, effectiveLimits); return report; @@ -123,7 +117,7 @@ public class DefaultSkillValidator implements SkillValidator { private static void validateName(Skill skill, SkillDocument document, SkillValidationReport report, SkillValidationMode mode) { SkillSourceLocation location = sourceLocation(document, "name"); - String name = skill.getName(); + String name = frontmatterString(document, "name"); if (isBlank(name)) { report.add(errorAt("NAME_REQUIRED", SkillPaths.SKILL_FILE, location, "Skill name is required.", "Set frontmatter name to a portable Skill name.")); @@ -153,13 +147,13 @@ public class DefaultSkillValidator implements SkillValidator { } } - private static void validateDescription(Skill skill, SkillDocument document, - SkillValidationReport report) { + private static void validateDescription(SkillDocument document, SkillValidationReport report) { SkillSourceLocation location = sourceLocation(document, "description"); - if (isBlank(skill.getDescription())) { + String description = frontmatterString(document, "description"); + if (isBlank(description)) { report.add(errorAt("DESCRIPTION_REQUIRED", SkillPaths.SKILL_FILE, location, "Skill description is required.", "Describe both the capability and when to use it.")); - } else if (skill.getDescription().length() > 1_024) { + } else if (description.length() > 1_024) { report.add(errorAt("DESCRIPTION_TOO_LONG", SkillPaths.SKILL_FILE, location, "Skill description cannot exceed 1024 characters.", "Shorten the description.")); } @@ -181,10 +175,8 @@ public class DefaultSkillValidator implements SkillValidator { } } - private static void validateDocument(Skill skill, SkillDocument document, SkillValidationReport report) { + private static void validateDocument(SkillDocument document, SkillValidationReport report) { Map values = document.getFrontmatter().getValues(); - validateCoreString(document, values, "name", skill.getName(), report); - validateCoreString(document, values, "description", skill.getDescription(), report); validateOptionalString(document, values, "license", null, report); Object compatibility = values.get("compatibility"); if (compatibility != null && (!(compatibility instanceof String text) @@ -195,12 +187,6 @@ public class DefaultSkillValidator implements SkillValidator { } validateOptionalString(document, values, "allowed-tools", "INVALID_ALLOWED_TOOLS", report); validateMetadataField(document, values.get("metadata"), report); - if (skill.getMetadata() == null || !skill.getMetadata().getValues().equals(values)) { - report.add(errorAt("METADATA_MISMATCH", SkillPaths.SKILL_FILE, - sourceLocation(document, "name"), - "Skill metadata must match SKILL.md frontmatter.", - "Reparse SKILL.md before saving the aggregate.")); - } if (document.getMarkdownBody().length() > 30_000) { report.add(new SkillValidationIssue("LONG_SKILL_BODY", SkillValidationSeverity.WARNING, SkillPaths.SKILL_FILE, null, null, @@ -209,21 +195,6 @@ public class DefaultSkillValidator implements SkillValidator { } } - private static void validateCoreString(SkillDocument document, Map values, - String key, String expected, - SkillValidationReport report) { - Object value = values.get(key); - if (!(value instanceof String text) || text.isBlank()) { - report.add(errorAt(key.toUpperCase(Locale.ROOT) + "_REQUIRED", SkillPaths.SKILL_FILE, - sourceLocation(document, key), - "SKILL.md frontmatter " + key + " must be a non-blank string.", null)); - } else if (!text.equals(expected)) { - report.add(errorAt(key.toUpperCase(Locale.ROOT) + "_MISMATCH", SkillPaths.SKILL_FILE, - sourceLocation(document, key), - "Skill " + key + " must match SKILL.md frontmatter.", null)); - } - } - private static void validateOptionalString(SkillDocument document, Map values, String key, String code, SkillValidationReport report) { @@ -335,11 +306,9 @@ public class DefaultSkillValidator implements SkillValidator { private static void validateResourceContent(SkillResource resource, String path, SkillValidationReport report, SkillPackageLimits limits) { - boolean expectedText = SkillResources.isText(path, resource.getKind(), resource.getMediaType()); + boolean expectedText = SkillResources.isText(path, resource.getMediaType()); if (expectedText != resource.isText()) { - String code = resource.getKind() == SkillResourceKind.SCRIPT - ? "SCRIPT_TEXT_REQUIRED" : "RESOURCE_CONTENT_MODE_MISMATCH"; - report.add(error(code, path, + report.add(error("RESOURCE_CONTENT_MODE_MISMATCH", path, expectedText ? "Resource path and media type require strict UTF-8 text content." : "Resource path and media type require a binary content reference.", @@ -350,13 +319,6 @@ public class DefaultSkillValidator implements SkillValidator { path, null, null, "Script resource is empty.", "Add script source or remove the unused file.")); } - if (resource.getKind() == SkillResourceKind.SCRIPT - && SkillScriptLanguage.fromPath(path) == SkillScriptLanguage.UNKNOWN) { - report.add(new SkillValidationIssue("SCRIPT_LANGUAGE_UNRECOGNIZED", - SkillValidationSeverity.WARNING, path, null, null, - "Script language is not recognized from its extension.", - "Use .py, .js, or .sh for first-class editing and syntax highlighting.")); - } if (isBlank(resource.getMediaType())) { report.add(error("MEDIA_TYPE_REQUIRED", path, "Skill resource media type is required.", null)); } @@ -525,4 +487,12 @@ public class DefaultSkillValidator implements SkillValidator { private static boolean isBlank(String value) { return value == null || value.isBlank(); } + + private static String frontmatterString(SkillDocument document, String key) { + if (document == null) { + return null; + } + Object value = document.getFrontmatter().get(key); + return value instanceof String text ? text : null; + } } diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java index 68b9dee..ee4beaa 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java @@ -1,7 +1,6 @@ package com.easyagents.skill.codec; import com.easyagents.skill.exception.SkillPackageException; -import com.easyagents.skill.exception.SkillValidationException; import com.easyagents.skill.factory.SkillFactory; import com.easyagents.skill.model.Skill; import com.easyagents.skill.model.SkillPackage; @@ -14,7 +13,6 @@ import com.easyagents.skill.store.SkillContentStore; import com.easyagents.skill.store.memory.InMemorySkillContentStore; import com.easyagents.skill.util.SkillHashes; import com.easyagents.skill.util.SkillPaths; -import com.easyagents.skill.validation.SkillValidator; import org.apache.commons.compress.archivers.zip.UnixStat; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; @@ -69,7 +67,7 @@ public class ZipSkillPackageCodecTest { } /** - * 单目录包导入通用资源并保持旧资源视图。 + * 单目录包导入通用资源。 */ @Test public void decodeWrappedSingleSkillWithGenericResources() { @@ -86,17 +84,16 @@ public class ZipSkillPackageCodecTest { Assert.assertEquals(SkillPackageLayout.SINGLE_DIRECTORY, result.getSkillPackage().getLayout()); Skill skill = result.getSkillPackage().getSkills().get(0); - Assert.assertNull(skill.getId()); Assert.assertEquals("skill-a", skill.getPackageRoot()); Assert.assertEquals(5, skill.getResources().size()); Assert.assertTrue(skill.getResources().stream().anyMatch(resource -> resource.getKind() == SkillResourceKind.EXAMPLE)); Assert.assertTrue(skill.getResources().stream().anyMatch(resource -> resource.getKind() == SkillResourceKind.OTHER)); - Assert.assertEquals(1, skill.getReferences().size()); - Assert.assertEquals(1, skill.getScripts().size()); - Assert.assertEquals(1, skill.getAssets().size()); - Assert.assertTrue(store.exists(skill.getAssets().get(0).getContentRef())); + SkillResource binary = skill.getResources().stream() + .filter(resource -> resource.getKind() == SkillResourceKind.ASSET) + .findFirst().orElseThrow(); + Assert.assertTrue(store.exists(binary.getContentRef())); } /** @@ -148,7 +145,7 @@ public class ZipSkillPackageCodecTest { "nested-skill/SKILL.md", utf8(markdown) ))).getSkillPackage().getSkills().get(0); - Assert.assertTrue(skill.getMetadata().get("metadata") instanceof Map); + Assert.assertTrue(skill.getDocument().getFrontmatter().get("metadata") instanceof Map); Assert.assertEquals(markdown, skill.getSkillContent()); } @@ -469,6 +466,56 @@ public class ZipSkillPackageCodecTest { Assert.assertEquals(0, store.committedCount); } + /** + * 资源的文本或二进制表示由文件类型决定,不受顶层语义目录约束。 + */ + @Test + public void decodeContentModeIndependentlyFromSemanticDirectory() throws Exception { + InMemorySkillContentStore store = new InMemorySkillContentStore(); + byte[] opaqueScript = new byte[]{0, (byte) 0xFF, 1}; + SkillPackageReadResult result = decode(new ZipSkillPackageCodec(store), zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/template.md", utf8("# Template"), + "skill-a/scripts/helper.bin", opaqueScript + ))); + + List resources = result.getSkillPackage().getSkills().get(0).getResources(); + SkillResource template = resources.stream() + .filter(resource -> "assets/template.md".equals(resource.getPath())) + .findFirst().orElseThrow(); + SkillResource helper = resources.stream() + .filter(resource -> "scripts/helper.bin".equals(resource.getPath())) + .findFirst().orElseThrow(); + + Assert.assertTrue(template.isText()); + Assert.assertEquals("# Template", template.getTextContent()); + Assert.assertFalse(helper.isText()); + try (InputStream input = store.open(helper.getContentRef())) { + Assert.assertArrayEquals(opaqueScript, input.readAllBytes()); + } + } + + /** + * 预检回滚首次失败时,异常清理路径会重试尚未释放的 stage。 + */ + @Test + public void retryTransientReportOnlyRollbackFailure() { + FlakyRollbackContentStore store = new FlakyRollbackContentStore(); + + try { + new ZipSkillPackageCodec(store).decode( + new ByteArrayInputStream(zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-a/assets/data.bin", new byte[]{1, 2, 3} + ))), SkillPackageReadOptions.reportOnly()); + Assert.fail("Transient rollback failure should remain visible to the caller."); + } catch (SkillPackageException expected) { + Assert.assertEquals("SKILL_CONTENT_ROLLBACK_ERROR", expected.getCode()); + Assert.assertEquals(2, store.rollbackAttempts); + Assert.assertTrue(store.rollbackCompleted); + } + } + /** * 多 Skill 预检报告为每个相对路径补齐各自根目录。 */ @@ -493,12 +540,14 @@ public class ZipSkillPackageCodecTest { */ @Test public void exportPrefixesValidationPathsForMultipleSkills() { - Skill first = SkillFactory.create("first", skillMd("skill-a")); + Skill first = SkillFactory.create(skillMd("skill-a")); first.setPackageRoot("skill-a"); - first.setDescription(null); - Skill second = SkillFactory.create("second", skillMd("skill-b")); + first.getDocument().removeFrontmatter("description"); + first.setDocument(first.getDocument()); + Skill second = SkillFactory.create(skillMd("skill-b")); second.setPackageRoot("skill-b"); - second.setDescription(null); + second.getDocument().removeFrontmatter("description"); + second.setDocument(second.getDocument()); ByteArrayOutputStream output = new ByteArrayOutputStream(); try { @@ -596,8 +645,6 @@ public class ZipSkillPackageCodecTest { "skill-a/references/a.md", utf8("# A"), "skill-a/assets/data.bin", new byte[]{1, 2, 3} ))); - first.getSkillPackage().getSkills().get(0).setId("internal-repository-id"); - ByteArrayOutputStream firstZip = new ByteArrayOutputStream(); SkillPackageWriteResult firstWrite = codec.encode(first.getSkillPackage(), firstZip, SkillPackageWriteOptions.defaults()); @@ -609,13 +656,11 @@ public class ZipSkillPackageCodecTest { Assert.assertEquals(firstWrite.getPackageHash(), secondWrite.getPackageHash()); Assert.assertEquals(SkillPackageLayout.SINGLE_DIRECTORY, firstWrite.getLayout()); Assert.assertEquals(SkillHashes.sha256Hex(firstZip.toByteArray()), firstWrite.getPackageHash()); - Assert.assertFalse(zipContains(firstZip.toByteArray(), "internal-repository-id")); - SkillPackageReadResult roundTrip = decode(codec, firstZip.toByteArray()); Skill original = first.getSkillPackage().getSkills().get(0); Skill decoded = roundTrip.getSkillPackage().getSkills().get(0); - Assert.assertNull(decoded.getId()); - Assert.assertEquals(original.getMetadata().getValues(), decoded.getMetadata().getValues()); + Assert.assertEquals(original.getDocument().getFrontmatter().getValues(), + decoded.getDocument().getFrontmatter().getValues()); Assert.assertEquals(original.getSkillContent(), decoded.getSkillContent()); Assert.assertEquals(original.getResources().stream().map(resource -> resource.getPath()).toList(), decoded.getResources().stream().map(resource -> resource.getPath()).toList()); @@ -630,7 +675,7 @@ public class ZipSkillPackageCodecTest { @Test public void highCompressionTextRoundTripsWithSameLimits() { String content = skillMd("skill-a") + "a".repeat(20_000); - Skill skill = SkillFactory.create("id", content); + Skill skill = SkillFactory.create(content); SkillPackage skillPackage = new SkillPackage( SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)); SkillPackageLimits limits = SkillPackageLimits.builder() @@ -678,7 +723,7 @@ public class ZipSkillPackageCodecTest { @Test public void enforceOutputLimitsOnGeneratedSkillEntryPath() { ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); - Skill skill = SkillFactory.create("id", skillMd("skill-a")); + Skill skill = SkillFactory.create(skillMd("skill-a")); String skillPath = "skill-a/SKILL.md"; assertExportIssue(codec, skill, new SkillPackageWriteOptions( @@ -690,95 +735,17 @@ public class ZipSkillPackageCodecTest { } /** - * 删除最后一个正式资源后不得从滞留的旧兼容视图恢复并再次导出。 + * Codec 内建的 YAML 与包大小安全校验不可绕过。 */ @Test - public void emptyCanonicalResourcesDoNotFallBackToLegacyViews() { - String content = "# Rules"; - SkillResource reference = new SkillResource(); - reference.setPath("references/rules.md"); - reference.setKind(SkillResourceKind.REFERENCE); - reference.setMediaType("text/markdown"); - reference.setTextContent(content); - reference.setContentHash(SkillHashes.sha256Hex(utf8(content))); - reference.setSize(utf8(content).length); - Skill skill = SkillFactory.createWithResources( - "id", skillMd("skill-a"), List.of(reference)); - Assert.assertEquals(1, skill.getReferences().size()); - skill.getResources().clear(); - + public void enforceMandatoryYamlAndPackageLimits() { ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)), - output, SkillPackageWriteOptions.defaults()); - Skill decoded = codec.decode(new ByteArrayInputStream(output.toByteArray()), - SkillPackageReadOptions.defaults()).getSkillPackage().getSkills().get(0); - - Assert.assertTrue(decoded.getResources().isEmpty()); - Assert.assertTrue(decoded.getReferences().isEmpty()); - } - - /** - * 注入的默认校验器仍属于附加校验,其更严格限额不得被类型判断或操作选项绕过。 - */ - @Test - public void injectedDefaultValidatorAppliesItsStricterLimits() { - ZipSkillPackageCodec codec = new ZipSkillPackageCodec( - new InMemorySkillContentStore(), - new com.easyagents.skill.validation.defaults.DefaultSkillValidator( - SkillPackageLimits.builder().maxTextFileBytes(128).build())); - Skill skill = SkillFactory.create("id", skillMd("skill-a") + "x".repeat(300)); - - assertExportIssue(codec, skill, SkillPackageWriteOptions.defaults(), - "TEXT_FILE_SIZE_LIMIT", SkillPaths.SKILL_FILE); - } - - /** - * 自定义限额感知校验器必须收到当前 Codec 操作选项。 - */ - @Test - public void customValidatorReceivesOperationLimits() { - SkillPackageLimits limits = SkillPackageLimits.builder().maxPathLength(64).build(); - SkillPackageLimits[] observed = new SkillPackageLimits[1]; - SkillValidator validator = new SkillValidator() { - @Override - public void validate(Skill skill) { - // 限额感知实现通过下方结构化入口完成业务校验。 - } - - @Override - public com.easyagents.skill.validation.SkillValidationReport validateReport( - Skill skill, SkillPackageLimits operationLimits) { - observed[0] = operationLimits; - return new com.easyagents.skill.validation.SkillValidationReport(); - } - }; - ZipSkillPackageCodec codec = new ZipSkillPackageCodec( - new InMemorySkillContentStore(), validator); - - codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, - List.of(SkillFactory.create("id", skillMd("skill-a")))), - new ByteArrayOutputStream(), new SkillPackageWriteOptions(limits)); - - Assert.assertSame(limits, observed[0]); - } - - /** - * noop 业务校验器不能绕过 Codec 内建的 YAML 与包大小安全校验。 - */ - @Test - public void noopValidatorCannotBypassMandatoryYamlAndPackageLimits() { - SkillValidator noopValidator = skill -> { - // 业务层不增加规则,Codec 仍必须独立完成标准安全校验。 - }; - ZipSkillPackageCodec codec = new ZipSkillPackageCodec( - new InMemorySkillContentStore(), noopValidator); - Skill malformedYaml = SkillFactory.create("id", skillMd("skill-a")); + Skill malformedYaml = SkillFactory.create(skillMd("skill-a")); malformedYaml.setSkillContent("---\nname: skill-a\ndescription: [\n---\n# Invalid\n"); assertExportIssue(codec, malformedYaml, SkillPackageWriteOptions.defaults(), "INVALID_FRONTMATTER_YAML", SkillPaths.SKILL_FILE); - Skill oversized = SkillFactory.create("id", skillMd("skill-a") + "x".repeat(300)); + Skill oversized = SkillFactory.create(skillMd("skill-a") + "x".repeat(300)); SkillPackageLimits limits = SkillPackageLimits.builder() .maxTextFileBytes(128) .build(); @@ -786,22 +753,6 @@ public class ZipSkillPackageCodecTest { "TEXT_FILE_SIZE_LIMIT", SkillPaths.SKILL_FILE); } - /** - * Codec 强制标准校验后仍执行调用方注入的业务校验器。 - */ - @Test - public void preserveAdditionalBusinessValidator() { - SkillValidator businessValidator = skill -> { - throw new SkillValidationException("CUSTOM_POLICY", SkillPaths.SKILL_FILE, - null, null, "Custom Skill policy rejected the package.", null); - }; - ZipSkillPackageCodec codec = new ZipSkillPackageCodec( - new InMemorySkillContentStore(), businessValidator); - - assertExportIssue(codec, SkillFactory.create("id", skillMd("skill-a")), - SkillPackageWriteOptions.defaults(), "CUSTOM_POLICY", SkillPaths.SKILL_FILE); - } - /** * SKILL.md 与普通文本资源中的孤立 UTF-16 surrogate 均以结构化错误拒绝。 */ @@ -809,12 +760,12 @@ public class ZipSkillPackageCodecTest { public void rejectMalformedUtf16SurrogateDuringExport() { String isolatedSurrogate = String.valueOf((char) 0xD800); ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); - Skill invalidDocument = SkillFactory.create("id", skillMd("skill-a")); + Skill invalidDocument = SkillFactory.create(skillMd("skill-a")); invalidDocument.setSkillContent(skillMd("skill-a") + isolatedSurrogate); assertExportIssue(codec, invalidDocument, SkillPackageWriteOptions.defaults(), "INVALID_UTF8", SkillPaths.SKILL_FILE); - Skill invalidResource = SkillFactory.create("id", skillMd("skill-a")); + Skill invalidResource = SkillFactory.create(skillMd("skill-a")); SkillResource resource = new SkillResource(); resource.setPath("references/invalid.md"); resource.setKind(SkillResourceKind.REFERENCE); @@ -826,7 +777,7 @@ public class ZipSkillPackageCodecTest { assertExportIssue(codec, invalidResource, SkillPackageWriteOptions.defaults(), "INVALID_UTF8", "references/invalid.md"); - Skill invalidPath = SkillFactory.create("id", skillMd("skill-a")); + Skill invalidPath = SkillFactory.create(skillMd("skill-a")); SkillResource pathResource = new SkillResource(); String malformedPath = "references/" + isolatedSurrogate + ".md"; pathResource.setPath(malformedPath); @@ -845,7 +796,7 @@ public class ZipSkillPackageCodecTest { */ @Test public void rejectMissingResourcePathBeforeExportWritesBytes() { - Skill skill = SkillFactory.create("id", skillMd("skill-a")); + Skill skill = SkillFactory.create(skillMd("skill-a")); SkillResource resource = new SkillResource(); resource.setKind(SkillResourceKind.REFERENCE); resource.setMediaType("text/markdown"); @@ -902,7 +853,7 @@ public class ZipSkillPackageCodecTest { */ @Test public void aggregateInvalidSkillBeforePreparingOutput() { - Skill skill = com.easyagents.skill.factory.SkillFactory.create("id", skillMd("skill-a")); + Skill skill = SkillFactory.create(skillMd("skill-a")); skill.setSkillContent(null); ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -975,7 +926,7 @@ public class ZipSkillPackageCodecTest { resource.setContentRef("sha256:" + hash); resource.setContentHash(hash); resource.setSize(expected.length); - Skill skill = com.easyagents.skill.factory.SkillFactory.create("id", skillMd("skill-a")); + Skill skill = SkillFactory.create(skillMd("skill-a")); skill.setResources(List.of(resource)); ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -1258,21 +1209,6 @@ public class ZipSkillPackageCodecTest { } } - private static boolean zipContains(byte[] zip, String value) { - try (ZipInputStream input = new ZipInputStream(new ByteArrayInputStream(zip), StandardCharsets.UTF_8)) { - ZipEntry entry; - while ((entry = input.getNextEntry()) != null) { - if (entry.getName().contains(value) - || new String(input.readAllBytes(), StandardCharsets.UTF_8).contains(value)) { - return true; - } - } - return false; - } catch (Exception e) { - throw new IllegalStateException(e); - } - } - private static byte[] utf8(String value) { return value.getBytes(StandardCharsets.UTF_8); } @@ -1285,7 +1221,7 @@ public class ZipSkillPackageCodecTest { return "---\nname: " + name + "\ndescription: ''\n---\n# " + name + "\n"; } - private static final class CountingContentStore implements SkillContentStore { + private static final class CountingContentStore extends InMemorySkillContentStore { private int putCount; @@ -1300,11 +1236,6 @@ public class ZipSkillPackageCodecTest { throw new UnsupportedOperationException(); } - @Override - public byte[] readAllBytes(String contentRef) { - throw new UnsupportedOperationException(); - } - @Override public boolean exists(String contentRef) { return false; @@ -1356,6 +1287,12 @@ public class ZipSkillPackageCodecTest { stagedCount--; } + @Override + public void retain(String contentRef) { + delegate.retain(contentRef); + committedCount++; + } + @Override public void release(String contentRef) { delegate.release(contentRef); @@ -1367,18 +1304,13 @@ public class ZipSkillPackageCodecTest { return delegate.open(contentRef); } - @Override - public byte[] readAllBytes(String contentRef) { - return delegate.readAllBytes(contentRef); - } - @Override public boolean exists(String contentRef) { return delegate.exists(contentRef); } } - private static final class CorruptReadContentStore implements SkillContentStore { + private static final class CorruptReadContentStore extends InMemorySkillContentStore { @Override public String put(byte[] bytes) { @@ -1390,14 +1322,25 @@ public class ZipSkillPackageCodecTest { return new ByteArrayInputStream(utf8("xyz")); } - @Override - public byte[] readAllBytes(String contentRef) { - return utf8("xyz"); - } - @Override public boolean exists(String contentRef) { return true; } } + + private static final class FlakyRollbackContentStore extends InMemorySkillContentStore { + + private int rollbackAttempts; + private boolean rollbackCompleted; + + @Override + public void rollback(SkillContentStage stage) { + rollbackAttempts++; + if (rollbackAttempts == 1) { + throw new IllegalStateException("simulated transient rollback failure"); + } + super.rollback(stage); + rollbackCompleted = true; + } + } } diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java deleted file mode 100644 index b2371a9..0000000 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/repository/memory/InMemorySkillRepositoryTest.java +++ /dev/null @@ -1,118 +0,0 @@ -package com.easyagents.skill.repository.memory; - -import com.easyagents.skill.model.Skill; -import com.easyagents.skill.model.SkillDescriptor; -import com.easyagents.skill.model.SkillReference; -import com.easyagents.skill.util.SkillResources; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Optional; - -/** - * InMemorySkillRepository 单元测试。 - */ -public class InMemorySkillRepositoryTest { - - /** - * 覆盖保存、读取、描述、列表、删除和存在性判断。 - */ - @Test - public void crudSkill() { - InMemorySkillRepository repository = new InMemorySkillRepository(); - Skill skill = skill(); - - repository.save(skill); - - Assert.assertTrue(repository.exists("skill-a")); - Assert.assertTrue(repository.get("skill-a").isPresent()); - Assert.assertTrue(repository.getDescriptor("skill-a").isPresent()); - Assert.assertEquals(1, repository.listDescriptors().size()); - - repository.delete("skill-a"); - - Assert.assertFalse(repository.exists("skill-a")); - Assert.assertFalse(repository.get("skill-a").isPresent()); - } - - /** - * descriptor 不携带 references/scripts/assets 内容。 - */ - @Test - public void descriptorDoesNotExposeResourceContent() { - InMemorySkillRepository repository = new InMemorySkillRepository(); - repository.save(skill()); - - Optional descriptor = repository.getDescriptor("skill-a"); - - Assert.assertTrue(descriptor.isPresent()); - Assert.assertEquals("skill-a", descriptor.get().getId()); - Assert.assertEquals("Skill A", descriptor.get().getName()); - } - - /** - * 读取返回副本,避免外部修改仓储内部状态。 - */ - @Test - public void getReturnsCopy() { - InMemorySkillRepository repository = new InMemorySkillRepository(); - repository.save(skill()); - - Skill loaded = repository.get("skill-a").get(); - loaded.setName("Changed"); - loaded.getReferences().get(0).setContent("changed"); - - Skill reloaded = repository.get("skill-a").get(); - Assert.assertEquals("Skill A", reloaded.getName()); - Assert.assertEquals("# A", reloaded.getReferences().get(0).getContent()); - } - - /** - * 仓储复制旧资源对象时应先迁移正式资源,不能因空 canonical 列表丢失 reference。 - */ - @Test - public void repositoryCopyMigratesLegacyOnlyResources() { - InMemorySkillRepository repository = new InMemorySkillRepository(); - Skill legacy = skill(); - Assert.assertFalse(legacy.isResourcesInitialized()); - - repository.save(legacy); - Skill loaded = repository.get("skill-a").orElseThrow(); - - Assert.assertTrue(loaded.isResourcesInitialized()); - Assert.assertEquals(1, SkillResources.canonicalResources(loaded).size()); - Assert.assertEquals("references/a.md", loaded.getResources().get(0).getPath()); - } - - /** - * 显式清空正式资源列表后,仓储往返不得从旧兼容视图恢复已删除资源。 - */ - @Test - public void repositoryCopyPreservesExplicitlyEmptyCanonicalResources() { - InMemorySkillRepository repository = new InMemorySkillRepository(); - Skill skill = skill(); - Assert.assertEquals(1, skill.getResources().size()); - skill.getResources().clear(); - - repository.save(skill); - Skill loaded = repository.get("skill-a").orElseThrow(); - - Assert.assertTrue(loaded.isResourcesInitialized()); - Assert.assertTrue(SkillResources.canonicalResources(loaded).isEmpty()); - } - - private static Skill skill() { - Skill skill = new Skill(); - skill.setId("skill-a"); - skill.setName("Skill A"); - skill.setDescription("Desc A"); - skill.setSkillContent("---\nname: Skill A\ndescription: Desc A\n---\n# Skill A\n"); - - SkillReference reference = new SkillReference(); - reference.setPath("references/a.md"); - reference.setName("a.md"); - reference.setContent("# A"); - skill.getReferences().add(reference); - return skill; - } -} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java index 43b4f38..a592f16 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/store/file/TemporaryFileSkillContentStoreTest.java @@ -4,7 +4,6 @@ import com.easyagents.skill.codec.ZipSkillPackageCodec; import com.easyagents.skill.exception.SkillException; import com.easyagents.skill.store.SkillContentStage; import com.easyagents.skill.store.SkillContentStore; -import com.easyagents.skill.store.memory.InMemorySkillContentStore; import com.easyagents.skill.util.SkillHashes; import org.junit.Assert; import org.junit.Test; @@ -167,7 +166,6 @@ public class TemporaryFileSkillContentStoreTest { ZipSkillPackageCodec defaultCodec = new ZipSkillPackageCodec(); SkillContentStore defaultStore = (SkillContentStore) contentStoreField.get(defaultCodec); Assert.assertTrue(defaultStore instanceof TemporaryFileSkillContentStore); - Assert.assertFalse(defaultStore instanceof InMemorySkillContentStore); Path defaultDirectory = ((TemporaryFileSkillContentStore) defaultStore).storageDirectory(); defaultCodec.close(); Assert.assertFalse(Files.exists(defaultDirectory)); diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java new file mode 100644 index 0000000..5b9e457 --- /dev/null +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStore.java @@ -0,0 +1,124 @@ +package com.easyagents.skill.store.memory; + +import com.easyagents.skill.exception.SkillException; +import com.easyagents.skill.store.SkillContentStage; +import com.easyagents.skill.store.SkillContentStore; +import com.easyagents.skill.util.SkillHashes; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * Codec 测试使用的内存内容存储。 + */ +public class InMemorySkillContentStore implements SkillContentStore { + + private final Map contents = new HashMap<>(); + private final Map stages = new HashMap<>(); + + @Override + public String put(byte[] bytes) { + byte[] copy = bytes == null ? new byte[0] : Arrays.copyOf(bytes, bytes.length); + String ref = SkillHashes.sha256Ref(copy); + StoredContent stored = contents.get(ref); + if (stored == null) { + contents.put(ref, new StoredContent(copy)); + } else { + stored.references++; + } + return ref; + } + + @Override + public SkillContentStage stage(InputStream inputStream, long maxBytes) { + byte[] bytes = readBounded(inputStream, maxBytes); + String stageId = UUID.randomUUID().toString(); + String hash = SkillHashes.sha256Hex(bytes); + stages.put(stageId, bytes); + return new SkillContentStage(stageId, "sha256:" + hash, hash, bytes.length); + } + + @Override + public String commit(SkillContentStage stage) { + byte[] bytes = stage == null ? null : stages.remove(stage.getStageId()); + if (bytes == null) { + throw new SkillException("Skill content stage does not exist."); + } + return put(bytes); + } + + @Override + public void rollback(SkillContentStage stage) { + if (stage != null) { + stages.remove(stage.getStageId()); + } + } + + @Override + public void retain(String contentRef) { + StoredContent stored = require(contentRef); + stored.references++; + } + + @Override + public void release(String contentRef) { + StoredContent stored = contents.get(contentRef); + if (stored != null && --stored.references <= 0) { + contents.remove(contentRef); + } + } + + @Override + public InputStream open(String contentRef) { + StoredContent stored = require(contentRef); + return new ByteArrayInputStream(Arrays.copyOf(stored.bytes, stored.bytes.length)); + } + + @Override + public boolean exists(String contentRef) { + return contents.containsKey(contentRef); + } + + private StoredContent require(String contentRef) { + StoredContent stored = contents.get(contentRef); + if (stored == null) { + throw new SkillException("Skill content does not exist: " + contentRef); + } + return stored; + } + + private static byte[] readBounded(InputStream inputStream, long maxBytes) { + try { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8_192]; + long total = 0; + int read; + while ((read = inputStream.read(buffer)) >= 0) { + total += read; + if (total > maxBytes) { + throw new SkillException("Skill content exceeds limit."); + } + output.write(buffer, 0, read); + } + return output.toByteArray(); + } catch (IOException exception) { + throw new SkillException("Failed to read Skill content.", exception); + } + } + + private static final class StoredContent { + + private final byte[] bytes; + private int references = 1; + + private StoredContent(byte[] bytes) { + this.bytes = bytes; + } + } +} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java deleted file mode 100644 index e5f14d3..0000000 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/store/memory/InMemorySkillContentStoreTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.easyagents.skill.store.memory; - -import com.easyagents.skill.store.SkillContentStage; -import org.junit.Assert; -import org.junit.Test; - -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -/** - * InMemorySkillContentStore 单元测试。 - */ -public class InMemorySkillContentStoreTest { - - /** - * put 返回 sha256 内容引用,且相同内容引用相同。 - */ - @Test - public void putReturnsStableSha256Ref() { - InMemorySkillContentStore store = new InMemorySkillContentStore(); - - String firstRef = store.put("abc".getBytes(StandardCharsets.UTF_8)); - String secondRef = store.put("abc".getBytes(StandardCharsets.UTF_8)); - - Assert.assertTrue(firstRef.startsWith("sha256:")); - Assert.assertEquals(firstRef, secondRef); - } - - /** - * open、readAllBytes 和 exists 正常工作。 - * - * @throws Exception 读取流失败时抛出 - */ - @Test - public void openReadAndExists() throws Exception { - InMemorySkillContentStore store = new InMemorySkillContentStore(); - byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); - - String contentRef = store.put(bytes); - - Assert.assertTrue(store.exists(contentRef)); - Assert.assertArrayEquals(bytes, store.readAllBytes(contentRef)); - try (InputStream inputStream = store.open(contentRef)) { - Assert.assertArrayEquals(bytes, inputStream.readAllBytes()); - } - } - - /** - * 暂存内容在 commit 前不可见,rollback 后不残留。 - */ - @Test - public void stageCommitAndRollback() { - InMemorySkillContentStore store = new InMemorySkillContentStore(); - byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); - - SkillContentStage rolledBack = store.stage(new java.io.ByteArrayInputStream(bytes), bytes.length); - Assert.assertFalse(store.exists(rolledBack.getContentRef())); - store.rollback(rolledBack); - Assert.assertFalse(store.exists(rolledBack.getContentRef())); - - SkillContentStage committed = store.stage(new java.io.ByteArrayInputStream(bytes), bytes.length); - String contentRef = store.commit(committed); - Assert.assertTrue(store.exists(contentRef)); - Assert.assertArrayEquals(bytes, store.readAllBytes(contentRef)); - } - - /** - * 相同内容按引用计数释放,归零后删除。 - */ - @Test - public void releaseDeletesOnlyAfterReferenceCountReachesZero() { - InMemorySkillContentStore store = new InMemorySkillContentStore(); - byte[] bytes = "abc".getBytes(StandardCharsets.UTF_8); - - String first = store.put(bytes); - String second = store.put(bytes); - store.release(first); - - Assert.assertTrue(store.exists(second)); - store.release(second); - Assert.assertFalse(store.exists(second)); - } -} diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java index 555e83d..95016f8 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillResourcesTest.java @@ -1,6 +1,5 @@ package com.easyagents.skill.util; -import com.easyagents.skill.model.SkillResourceKind; import org.junit.Assert; import org.junit.Test; @@ -15,8 +14,18 @@ public class SkillResourcesTest { @Test public void recognizeBothHtmlExtensionsAsText() { Assert.assertTrue(SkillResources.isText( - "references/page.htm", SkillResourceKind.REFERENCE, "application/octet-stream")); + "references/page.htm", "application/octet-stream")); Assert.assertTrue(SkillResources.isText( - "references/page.html", SkillResourceKind.REFERENCE, "application/octet-stream")); + "references/page.html", "application/octet-stream")); + } + + /** + * 资源目录只表达语义分类,不决定文本或二进制存储。 + */ + @Test + public void determineContentModeIndependentlyFromTopLevelDirectory() { + Assert.assertTrue(SkillResources.isText("assets/template.md", "application/octet-stream")); + Assert.assertTrue(SkillResources.isText("custom/data.csv", "application/octet-stream")); + Assert.assertFalse(SkillResources.isText("scripts/helper.bin", "application/octet-stream")); } } diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java index 7af188f..ad91028 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/validation/defaults/DefaultSkillValidatorTest.java @@ -45,15 +45,15 @@ public class DefaultSkillValidatorTest { String content = "---\nname: nested-skill\ndescription: Handles nested metadata\n" + "metadata:\n enabled: true\n retries: 3\n tags:\n - alpha\n - beta\n" + "---\n# Nested\n"; - Skill skill = SkillFactory.create("repository-id", content); + Skill skill = SkillFactory.create(content); validator.validate(skill); - Assert.assertTrue(skill.getMetadata().get("metadata") instanceof java.util.Map); + Assert.assertTrue(skill.getDocument().getFrontmatter().get("metadata") instanceof java.util.Map); } /** - * 结构化文档重新应用到聚合时同步 name、description 和 metadata。 + * 结构化文档更新后名称与描述从 frontmatter 派生。 */ @Test public void applyEditedDocumentToAggregate() { @@ -64,7 +64,8 @@ public class DefaultSkillValidatorTest { skill.setDocument(document); Assert.assertEquals("Updated description for the Skill", skill.getDescription()); - Assert.assertEquals("Updated description for the Skill", skill.getMetadata().get("description")); + Assert.assertEquals("Updated description for the Skill", + skill.getDocument().getFrontmatter().get("description")); validator.validate(skill); } @@ -98,20 +99,12 @@ public class DefaultSkillValidatorTest { && issue.getSeverity() == SkillValidationSeverity.ERROR)); } - /** - * 严格工厂入口执行正式标准名称校验。 - */ - @Test(expected = SkillValidationException.class) - public void strictFactoryRejectsLegacyUnderscoreName() { - SkillFactory.createStrict("repository-id", skillMd("legacy_skill")); - } - /** * AgentScope 嵌套 metadata 保真并标记标准兼容 warning。 */ @Test public void nestedMetadataProducesCompatibilityWarning() { - Skill skill = SkillFactory.create("id", "---\nname: metadata-skill\n" + Skill skill = SkillFactory.create("---\nname: metadata-skill\n" + "description: Nested metadata compatibility\nmetadata:\n provider:\n enabled: true\n" + "---\n# Metadata\n"); @@ -127,7 +120,7 @@ public class DefaultSkillValidatorTest { */ @Test public void rejectInvalidOptionalStandardFields() { - Skill skill = SkillFactory.create("id", "---\nname: optional-skill\n" + Skill skill = SkillFactory.create("---\nname: optional-skill\n" + "description: Invalid optional fields\nlicense: []\ncompatibility: ''\nallowed-tools:\n - Read\n" + "---\n# Optional\n"); @@ -261,10 +254,10 @@ public class DefaultSkillValidatorTest { } /** - * scripts 目录只接受严格 UTF-8 文本表示,二进制引用必须被拒绝。 + * 文本扩展名必须使用严格 UTF-8 文本表示,与所在目录无关。 */ @Test - public void rejectBinaryScriptResource() { + public void rejectBinaryRepresentationForTextExtension() { Skill skill = validSkill("skill-a"); SkillResource script = binaryResource("scripts/run.sh", "echo unsafe"); script.setKind(SkillResourceKind.SCRIPT); @@ -273,23 +266,37 @@ public class DefaultSkillValidatorTest { SkillValidationReport report = validator.validateReport(skill); Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> - "SCRIPT_TEXT_REQUIRED".equals(issue.getCode()) + "RESOURCE_CONTENT_MODE_MISMATCH".equals(issue.getCode()) && issue.getSeverity() == SkillValidationSeverity.ERROR)); } /** - * 资源的文本或二进制表示必须与统一路径和媒体类型判定一致。 + * assets 目录中的文本资源使用规范文本表示。 */ @Test - public void rejectNonCanonicalResourceContentMode() { + public void acceptTextResourceInAssetsDirectory() { Skill skill = validSkill("skill-a"); SkillResource textAsset = textResource("assets/readme.txt", SkillResourceKind.ASSET, "text asset"); skill.setResources(java.util.List.of(textAsset)); SkillValidationReport report = validator.validateReport(skill); - Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> - "RESOURCE_CONTENT_MODE_MISMATCH".equals(issue.getCode()))); + Assert.assertFalse(report.hasErrors()); + } + + /** + * scripts 目录允许保存不透明二进制辅助文件。 + */ + @Test + public void acceptBinaryResourceInScriptsDirectory() { + Skill skill = validSkill("skill-a"); + SkillResource binaryScript = binaryResource("scripts/helper.bin", "opaque"); + binaryScript.setKind(SkillResourceKind.SCRIPT); + skill.setResources(java.util.List.of(binaryScript)); + + SkillValidationReport report = validator.validateReport(skill); + + Assert.assertFalse(report.hasErrors()); } /** @@ -309,23 +316,6 @@ public class DefaultSkillValidatorTest { && issue.getSeverity() == SkillValidationSeverity.WARNING)); } - /** - * 未识别脚本语言需要提示,但不能破坏外部标准 Skill 包的资源保真。 - */ - @Test - public void warnForUnrecognizedScriptLanguageExtension() { - Skill skill = validSkill("skill-a"); - skill.setResources(java.util.List.of( - textResource("scripts/run.txt", SkillResourceKind.SCRIPT, "echo ok"))); - - SkillValidationReport report = validator.validateReport(skill); - - Assert.assertFalse(report.hasErrors()); - Assert.assertTrue(report.getIssues().stream().anyMatch(issue -> - "SCRIPT_LANGUAGE_UNRECOGNIZED".equals(issue.getCode()) - && issue.getSeverity() == SkillValidationSeverity.WARNING)); - } - /** * 结构化校验器执行资源单文件安全限额。 */ @@ -361,19 +351,8 @@ public class DefaultSkillValidatorTest { Assert.assertFalse(report.hasErrors()); } - /** - * 元数据与 SKILL.md 不一致时失败。 - */ - @Test(expected = SkillValidationException.class) - public void rejectMetadataMismatch() { - Skill skill = validSkill("skill-a"); - skill.getMetadata().put("extra", "value"); - - validator.validate(skill); - } - private static Skill validSkill(String name) { - return SkillFactory.create("repository-id", skillMd(name)); + return SkillFactory.create(skillMd(name)); } private static String skillMd(String name) { From 49a7de34bb3008d97eaca8e57c37fbeeb323d958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 19 Aug 2026 21:50:58 +0800 Subject: [PATCH 28/33] =?UTF-8?q?feat:=20=E6=A0=87=E5=87=86=E5=8C=96=20Age?= =?UTF-8?q?nt=20AG-UI=20=E4=B8=8E=E5=AE=A1=E6=89=B9=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 AG-UI 事件投影与协议编码模块 - 支持 Turn 级审批作用域和受信任动态审批策略 --- .../interceptor/ToolHitlInterceptor.java | 192 +++++++++++++- .../hitl/AgentToolApprovalCoordinator.java | 87 +++++- .../hitl/AgentToolApprovalEvaluation.java | 49 ++++ .../runtime/hitl/AgentToolApprovalPolicy.java | 18 ++ .../agent/runtime/tool/AgentToolSpec.java | 20 ++ .../AgentToolApprovalCoordinatorTest.java | 132 ++++++++++ easy-agents-agui/pom.xml | 36 +++ .../easyagents/agui/AguiExtendedEvent.java | 84 ++++++ .../agui/AguiProtocolEventEncoder.java | 39 +++ .../agui/AguiRuntimeEventProjector.java | 249 ++++++++++++++++++ .../agui/AguiProtocolEventEncoderTest.java | 43 +++ .../agui/AguiRuntimeEventProjectorTest.java | 90 +++++++ easy-agents-bom/pom.xml | 4 + pom.xml | 13 + 14 files changed, 1043 insertions(+), 13 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalEvaluation.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalPolicy.java create mode 100644 easy-agents-agui/pom.xml create mode 100644 easy-agents-agui/src/main/java/com/easyagents/agui/AguiExtendedEvent.java create mode 100644 easy-agents-agui/src/main/java/com/easyagents/agui/AguiProtocolEventEncoder.java create mode 100644 easy-agents-agui/src/main/java/com/easyagents/agui/AguiRuntimeEventProjector.java create mode 100644 easy-agents-agui/src/test/java/com/easyagents/agui/AguiProtocolEventEncoderTest.java create mode 100644 easy-agents-agui/src/test/java/com/easyagents/agui/AguiRuntimeEventProjectorTest.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java index b72bbc2..83c705a 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/interceptor/ToolHitlInterceptor.java @@ -7,6 +7,7 @@ import com.easyagents.agent.runtime.event.AgentRuntimeEventBridge; import com.easyagents.agent.runtime.event.AgentRuntimeEventType; import com.easyagents.agent.runtime.event.AgentRuntimeInterceptor; import com.easyagents.agent.runtime.hitl.AgentPendingState; +import com.easyagents.agent.runtime.hitl.AgentToolApprovalEvaluation; import com.easyagents.agent.runtime.hitl.AgentToolApprovalCoordinator; import com.easyagents.agent.runtime.hitl.AgentToolApprovalRequest; import com.easyagents.agent.runtime.tool.AgentToolSpec; @@ -22,9 +23,11 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -140,7 +143,12 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { private void interceptPreActing(PreActingEvent event) { ToolUseBlock toolUse = event.getToolUse(); AgentToolSpec toolSpec = toolUse == null ? null : toolSpecs.get(toolUse.getName()); - if (toolSpec == null || !toolSpec.isApprovalRequired()) { + if (!requiresApproval(toolSpec, toolUse)) { + return; + } + Map approvalMetadata = approvalMetadata(toolSpec, toolUse); + if (!requiresForcedApproval(toolSpec, toolUse) + && approvalCoordinator.isReusableApprovalGranted(approvalMetadata)) { return; } // 执行授权与 toolCallId、工具名称及入参同时绑定,并且只能消费一次。 @@ -239,7 +247,121 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { */ private boolean isApprovalRequired(ToolUseBlock toolUse) { AgentToolSpec toolSpec = toolUse == null ? null : toolSpecs.get(toolUse.getName()); - return toolSpec != null && toolSpec.isApprovalRequired(); + return requiresApproval(toolSpec, toolUse); + } + + /** + * 判断工具声明或当前调用是否要求审批。 + * + * @param toolSpec 工具声明 + * @param toolUse 当前工具调用 + * @return 需要审批时为 true + */ + private boolean requiresApproval(AgentToolSpec toolSpec, ToolUseBlock toolUse) { + if (toolSpec == null) { + return false; + } + AgentToolApprovalEvaluation evaluation = approvalEvaluation(toolSpec, toolUse); + if (evaluation != null && !evaluation.valid()) { + // 无效命令直接进入工具并返回结构化拒绝,避免产生必然失败的审批请求。 + return false; + } + return toolSpec.isApprovalRequired() + || (evaluation != null && evaluation.approvalRequired()) + || requiresLegacyForcedApproval(toolSpec, toolUse); + } + + /** + * 根据工具声明中的强制审批命令规则检查当前调用。 + * + *

命令首词解析与受控 Shell 的引号、反斜杠规则保持一致,避免通过 + * {@code 'rm'} 或 {@code r\m} 绕过动态审批。畸形命令仍由 Shell 工具拒绝。

+ * + * @param toolSpec 工具声明 + * @param toolUse 当前工具调用 + * @return 命中强制审批命令时为 true + */ + private boolean requiresForcedApproval(AgentToolSpec toolSpec, ToolUseBlock toolUse) { + AgentToolApprovalEvaluation evaluation = approvalEvaluation(toolSpec, toolUse); + if (evaluation != null) { + return evaluation.valid() && evaluation.forced(); + } + return requiresLegacyForcedApproval(toolSpec, toolUse); + } + + /** + * 使用旧版元数据规则判断当前调用是否命中强制审批命令。 + * + * @param toolSpec 工具声明 + * @param toolUse 当前工具调用 + * @return 命中旧版强制审批规则时为 true + */ + private boolean requiresLegacyForcedApproval(AgentToolSpec toolSpec, ToolUseBlock toolUse) { + if (toolSpec == null || toolUse == null || toolSpec.getMetadata() == null) { + return false; + } + Object commandsValue = toolSpec.getMetadata().get("forceApprovalCommands"); + Object argumentValue = toolSpec.getMetadata().get("forceApprovalCommandArgument"); + if (!(commandsValue instanceof Iterable commands) || !(argumentValue instanceof String argumentName) + || argumentName.isBlank() || toolUse.getInput() == null) { + return false; + } + Object commandValue = toolUse.getInput().get(argumentName); + if (!(commandValue instanceof String command)) { + return false; + } + String executable = firstCommandToken(command); + if (executable == null) { + return false; + } + for (Object forcedCommand : commands) { + if (forcedCommand instanceof String value && executable.equals(value)) { + return true; + } + } + return false; + } + + /** + * 解析受限命令行的首个参数。 + * + * @param command 命令行 + * @return 首个参数;无有效参数时返回 null + */ + private String firstCommandToken(String command) { + if (command == null || command.isBlank()) { + return null; + } + StringBuilder token = new StringBuilder(); + char quote = 0; + boolean escaping = false; + boolean started = false; + for (int index = 0; index < command.length(); index++) { + char character = command.charAt(index); + if (!started && Character.isWhitespace(character)) { + continue; + } + started = true; + if (escaping) { + token.append(character); + escaping = false; + } else if (character == '\\' && quote != '\'') { + escaping = true; + } else if (character == '\'' || character == '"') { + if (quote == 0) { + quote = character; + } else if (quote == character) { + quote = 0; + } else { + token.append(character); + } + } else if (Character.isWhitespace(character) && quote == 0) { + break; + } else { + token.append(character); + } + } + return token.isEmpty() ? null : token.toString(); } /** @@ -270,9 +392,26 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { if (toolUses == null || toolUses.isEmpty()) { return List.of(); } - return toolUses.stream() - .filter(this::isApprovalRequired) - .toList(); + List approvalTools = new ArrayList<>(); + Set pendingReusableScopes = new LinkedHashSet<>(); + for (ToolUseBlock toolUse : toolUses) { + AgentToolSpec toolSpec = toolUse == null ? null : toolSpecs.get(toolUse.getName()); + if (!requiresApproval(toolSpec, toolUse)) { + continue; + } + Map approvalMetadata = approvalMetadata(toolSpec, toolUse); + if (!requiresForcedApproval(toolSpec, toolUse) + && approvalCoordinator.isReusableApprovalGranted(approvalMetadata)) { + continue; + } + String reusableScope = approvalCoordinator.reusableApprovalScope(approvalMetadata); + if (reusableScope != null && !pendingReusableScopes.add(reusableScope)) { + // 同一推理消息中同一 MCP 的多个工具共享一个审批请求。 + continue; + } + approvalTools.add(toolUse); + } + return approvalTools; } /** @@ -294,12 +433,18 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { Map metadata = approvalRequest == null ? new LinkedHashMap<>() : new LinkedHashMap<>(approvalRequest.getMetadata()); + metadata.putAll(toolUse.getMetadata() == null ? Map.of() : toolUse.getMetadata()); if (toolSpec.getMetadata() != null && !toolSpec.getMetadata().isEmpty()) { + // ToolSpec 由工具编译阶段生成,必须覆盖模型返回的同名治理字段(例如 toolType、mcpId)。 metadata.putAll(toolSpec.getMetadata()); } + AgentToolApprovalEvaluation evaluation = approvalEvaluation(toolSpec, toolUse); + if (evaluation != null && evaluation.metadata() != null) { + // 动态策略由受信任工具实例计算,必须覆盖模型与静态声明中的同名字段。 + metadata.putAll(evaluation.metadata()); + } metadata.put("phase", "POST_REASONING"); metadata.put("source", "TOOL_HITL_INTERCEPTOR"); - metadata.putAll(toolUse.getMetadata() == null ? Map.of() : toolUse.getMetadata()); return approvalCoordinator.register( context == null ? null : context.getSessionId(), context == null || context.getAgentDefinition() == null ? null : context.getAgentDefinition().getAgentId(), @@ -312,6 +457,40 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { approvalBatchId); } + /** + * 调用工具声明中的受信任动态审批策略。 + * + * @param toolSpec 工具声明 + * @param toolUse 当前工具调用 + * @return 动态审批判定;未配置策略时返回 null + */ + private AgentToolApprovalEvaluation approvalEvaluation(AgentToolSpec toolSpec, ToolUseBlock toolUse) { + if (toolSpec == null || toolSpec.getApprovalPolicy() == null || toolUse == null) { + return null; + } + return toolSpec.getApprovalPolicy().evaluate( + toolUse.getInput() == null ? Map.of() : toolUse.getInput()); + } + + /** + * 合并静态工具元数据与动态审批元数据。 + * + * @param toolSpec 工具声明 + * @param toolUse 当前工具调用 + * @return 用于审批作用域判断的受信任元数据 + */ + private Map approvalMetadata(AgentToolSpec toolSpec, ToolUseBlock toolUse) { + Map metadata = new LinkedHashMap<>(); + if (toolSpec != null && toolSpec.getMetadata() != null) { + metadata.putAll(toolSpec.getMetadata()); + } + AgentToolApprovalEvaluation evaluation = approvalEvaluation(toolSpec, toolUse); + if (evaluation != null && evaluation.metadata() != null) { + metadata.putAll(evaluation.metadata()); + } + return metadata; + } + /** * 构建工具审批请求事件。 * @@ -373,6 +552,7 @@ public class ToolHitlInterceptor implements AgentRuntimeInterceptor { putIfPresent(payload, metadata, "toolDisplayName"); putIfPresent(payload, metadata, "rawMcpToolName"); putIfPresent(payload, metadata, "mcpToolName"); + putIfPresent(payload, metadata, "mcpId"); putIfPresent(payload, metadata, "mcpName"); putIfPresent(payload, metadata, "mcpTitle"); } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java index db1ca3e..e3435ff 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinator.java @@ -19,6 +19,9 @@ import java.util.concurrent.CompletableFuture; */ public class AgentToolApprovalCoordinator { + /** MCP 工具类型。 */ + private static final String MCP_TOOL_TYPE = "MCP"; + /** 是否启用内存审批协调。 */ private final boolean enabled; /** 恢复令牌到待审批项的索引。 */ @@ -29,6 +32,8 @@ public class AgentToolApprovalCoordinator { private final Map tokensByToolCallId = new LinkedHashMap<>(); /** 工具调用ID到一次性执行授权的索引。 */ private final Map executionAuthorizations = new LinkedHashMap<>(); + /** 当前 Turn 已批准的可复用工具作用域。 */ + private final Set reusableApprovalScopes = new LinkedHashSet<>(); /** * 创建已启用的协调器。 @@ -300,11 +305,12 @@ public class AgentToolApprovalCoordinator { } /** - * 根据服务端持久化审批结果签发受信任的一次性执行授权。 + * 根据服务端持久化审批结果签发受信任的执行授权。 * *

恢复元数据应提供单个 {@code toolCallId/toolName/toolInput},或通过 - * {@code approvedToolCalls} 提供上述字段组成的列表。该入口仅供已经完成持久化令牌 - * 校验和一次性消费的服务端集成层使用。

+ * {@code approvedToolCalls} 提供上述字段组成的列表。MCP 调用可额外携带受信任的 + * {@code toolType/mcpId},用于签发当前 Turn 的复用作用域。该入口仅供已经完成 + * 持久化令牌校验和一次性消费的服务端集成层使用。

* * @param request 受信任恢复请求 */ @@ -318,16 +324,17 @@ public class AgentToolApprovalCoordinator { : request.getMetadata(); Object approvedToolCalls = metadata.get("approvedToolCalls"); Map trustedAuthorizations = new LinkedHashMap<>(); + Set trustedReusableScopes = new LinkedHashSet<>(); int authorizationCount = 0; if (approvedToolCalls instanceof List calls) { for (Object call : calls) { if (call instanceof Map callMap) { - authorizeTrustedCall(callMap, trustedAuthorizations); + authorizeTrustedCall(callMap, trustedAuthorizations, trustedReusableScopes); authorizationCount++; } } } else if (metadata.containsKey("toolCallId")) { - authorizeTrustedCall(metadata, trustedAuthorizations); + authorizeTrustedCall(metadata, trustedAuthorizations, trustedReusableScopes); authorizationCount++; } if (authorizationCount == 0) { @@ -335,6 +342,7 @@ public class AgentToolApprovalCoordinator { "Trusted resume metadata must include approved toolCallId, toolName, and toolInput."); } executionAuthorizations.putAll(trustedAuthorizations); + reusableApprovalScopes.addAll(trustedReusableScopes); } /** @@ -363,6 +371,45 @@ public class AgentToolApprovalCoordinator { } } + /** + * 判断工具元数据对应的复用作用域是否已在当前 Turn 获得批准。 + * + * @param metadata 服务端工具元数据 + * @return 当前 Turn 已批准时为 true + */ + public synchronized boolean isReusableApprovalGranted(Map metadata) { + String approvalScope = reusableApprovalScope(metadata); + return approvalScope != null && reusableApprovalScopes.contains(approvalScope); + } + + /** + * 解析可在当前 Turn 复用的审批作用域。 + * + *

MCP 使用稳定 {@code mcpId} 生成作用域;受控 Shell 脚本仅接受动态审批策略写入的 + * 内容摘要作用域。缺少受信任标识时返回 null,使调用方继续执行逐调用审批。

+ * + * @param metadata 服务端工具元数据 + * @return 可复用审批作用域;不可复用时返回 null + */ + public String reusableApprovalScope(Map metadata) { + if (metadata == null || metadata.isEmpty()) { + return null; + } + String explicitScope = stringValue(metadata.get("approvalScope")); + if (Boolean.TRUE.equals(metadata.get("operateTool")) + && "SHELL".equalsIgnoreCase(stringValue(metadata.get("operateToolType"))) + && explicitScope != null + && explicitScope.startsWith("SHELL_SCRIPT:")) { + return explicitScope; + } + String toolType = stringValue(metadata.get("toolType")); + String mcpId = stringValue(metadata.get("mcpId")); + if (!MCP_TOOL_TYPE.equalsIgnoreCase(toolType) || mcpId == null) { + return null; + } + return MCP_TOOL_TYPE + ":" + mcpId; + } + /** * 清理尚未消费的一次性执行授权。 */ @@ -370,6 +417,13 @@ public class AgentToolApprovalCoordinator { executionAuthorizations.clear(); } + /** + * 清理当前 Turn 的可复用工具审批作用域。 + */ + public synchronized void clearReusableApprovalScopes() { + reusableApprovalScopes.clear(); + } + /** * 获取指定会话当前仍待处理的审批状态。 * @@ -399,6 +453,7 @@ public class AgentToolApprovalCoordinator { approvals.clear(); tokensByToolCallId.clear(); executionAuthorizations.clear(); + reusableApprovalScopes.clear(); } /** @@ -459,6 +514,11 @@ public class AgentToolApprovalCoordinator { */ private void authorize(PendingApproval pendingApproval) { AgentPendingState state = pendingApproval.state; + String approvalScope = reusableApprovalScope(state.getMetadata()); + if (approvalScope != null) { + reusableApprovalScopes.add(approvalScope); + return; + } if (state.getToolCallId() == null || state.getToolCallId().isBlank()) { throw new AgentRuntimeException("Approved tool call is missing toolCallId."); } @@ -471,10 +531,12 @@ public class AgentToolApprovalCoordinator { * 为服务端持久化审批结果签发一次性执行授权。 * * @param callMap 已批准调用元数据 - * @param trustedAuthorizations 本次恢复待签发的临时授权集合 + * @param trustedAuthorizations 本次恢复待签发的一次性授权集合 + * @param trustedReusableScopes 本次恢复待签发的可复用作用域集合 */ private void authorizeTrustedCall(Map callMap, - Map trustedAuthorizations) { + Map trustedAuthorizations, + Set trustedReusableScopes) { String toolCallId = stringValue(callMap.get("toolCallId")); String toolName = stringValue(callMap.get("toolName")); if (toolCallId == null || toolName == null) { @@ -482,6 +544,17 @@ public class AgentToolApprovalCoordinator { "Trusted resume metadata must include non-empty toolCallId and toolName."); } Map toolInput = stringKeyMap(callMap.get("toolInput")); + Map scopeMetadata = new LinkedHashMap<>(); + scopeMetadata.put("toolType", callMap.get("toolType")); + scopeMetadata.put("mcpId", callMap.get("mcpId")); + scopeMetadata.put("operateTool", callMap.get("operateTool")); + scopeMetadata.put("operateToolType", callMap.get("operateToolType")); + scopeMetadata.put("approvalScope", callMap.get("approvalScope")); + String reusableScope = reusableApprovalScope(scopeMetadata); + if (reusableScope != null) { + trustedReusableScopes.add(reusableScope); + return; + } ExecutionAuthorization authorization = new ExecutionAuthorization(toolName, toolInput); ExecutionAuthorization previous = trustedAuthorizations.put(toolCallId, authorization); if (previous != null diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalEvaluation.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalEvaluation.java new file mode 100644 index 0000000..f95289e --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalEvaluation.java @@ -0,0 +1,49 @@ +package com.easyagents.agent.runtime.hitl; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * 单次工具调用的动态审批判定。 + * + * @param valid 调用是否通过审批前静态校验 + * @param approvalRequired 是否需要人工审批 + * @param forced 是否禁止复用既有审批 + * @param reusableScope 可复用审批作用域;为空表示逐调用审批 + * @param metadata 写入审批事件的受信任元数据 + */ +public record AgentToolApprovalEvaluation( + boolean valid, + boolean approvalRequired, + boolean forced, + String reusableScope, + Map metadata) { + + /** + * 创建审批前静态校验失败的判定。 + * + * @return 无需弹出审批的无效判定 + */ + public static AgentToolApprovalEvaluation invalid() { + return new AgentToolApprovalEvaluation(false, false, false, null, Map.of()); + } + + /** + * 创建通过静态校验的判定。 + * + * @param approvalRequired 是否需要审批 + * @param forced 是否强制逐调用审批 + * @param reusableScope 可复用作用域 + * @return 动态审批判定 + */ + public static AgentToolApprovalEvaluation valid(boolean approvalRequired, + boolean forced, + String reusableScope) { + Map metadata = new LinkedHashMap<>(); + if (reusableScope != null && !reusableScope.isBlank()) { + metadata.put("approvalScope", reusableScope); + } + return new AgentToolApprovalEvaluation( + true, approvalRequired, forced, reusableScope, Map.copyOf(metadata)); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalPolicy.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalPolicy.java new file mode 100644 index 0000000..7c1c1d3 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalPolicy.java @@ -0,0 +1,18 @@ +package com.easyagents.agent.runtime.hitl; + +import java.util.Map; + +/** + * 根据单次工具入参执行审批前校验并计算动态审批策略。 + */ +@FunctionalInterface +public interface AgentToolApprovalPolicy { + + /** + * 评估一次工具调用。 + * + * @param toolInput 工具调用入参 + * @return 动态审批判定 + */ + AgentToolApprovalEvaluation evaluate(Map toolInput); +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/AgentToolSpec.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/AgentToolSpec.java index 9755333..fd7757a 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/AgentToolSpec.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/AgentToolSpec.java @@ -1,6 +1,7 @@ package com.easyagents.agent.runtime.tool; import com.easyagents.agent.runtime.hitl.AgentToolApprovalRequest; +import com.easyagents.agent.runtime.hitl.AgentToolApprovalPolicy; import java.util.LinkedHashMap; import java.util.Map; @@ -18,6 +19,7 @@ public class AgentToolSpec { private AgentToolVisibility visibility = AgentToolVisibility.VISIBLE; private boolean approvalRequired; private AgentToolApprovalRequest approvalRequest = new AgentToolApprovalRequest(); + private AgentToolApprovalPolicy approvalPolicy; private Map metadata = new LinkedHashMap<>(); /** @@ -164,6 +166,24 @@ public class AgentToolSpec { this.approvalRequest = approvalRequest == null ? new AgentToolApprovalRequest() : approvalRequest; } + /** + * 获取单次调用动态审批策略。 + * + * @return 动态审批策略;未配置时返回 null + */ + public AgentToolApprovalPolicy getApprovalPolicy() { + return approvalPolicy; + } + + /** + * 设置单次调用动态审批策略。 + * + * @param approvalPolicy 动态审批策略 + */ + public void setApprovalPolicy(AgentToolApprovalPolicy approvalPolicy) { + this.approvalPolicy = approvalPolicy; + } + /** * 获取元数据。 * diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java index c3aad9d..f78a150 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/hitl/AgentToolApprovalCoordinatorTest.java @@ -158,6 +158,138 @@ public class AgentToolApprovalCoordinatorTest { coordinator, "call-1", "search", Map.of("q", "easyflow")); } + /** + * 验证受信任恢复可以签发当前 Turn 内可复用的 MCP 审批作用域。 + */ + @Test + public void shouldAuthorizeTrustedMcpScope() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentResumeRequest request = new AgentResumeRequest(); + AgentResumeToken token = new AgentResumeToken(); + token.setValue("persisted-token"); + request.setResumeToken(token); + request.setApproved(true); + request.setTrusted(true); + request.setMetadata(Map.of( + "toolCallId", "call-1", + "toolName", "mcp_101_search", + "toolInput", Map.of("q", "easyflow"), + "toolType", "MCP", + "mcpId", "101")); + + coordinator.authorizeTrustedExecution(request); + + Assert.assertTrue(coordinator.isReusableApprovalGranted( + Map.of("toolType", "MCP", "mcpId", "101"))); + } + + /** + * 验证 MCP 批准可在当前 Turn 按稳定 mcpId 复用。 + */ + @Test + public void shouldReuseApprovedMcpScopeWithinCurrentTurn() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + Map mcpMetadata = Map.of("toolType", "MCP", "mcpId", "101"); + AgentPendingState pending = coordinator.register( + "session-1", + "agent-1", + "call-resolve", + "mcp_101_resolve_library_id", + "approve", + Map.of("libraryName", "AG-UI"), + mcpMetadata, + Instant.now().plusSeconds(60), + "batch-mcp"); + + coordinator.resolve(resume(pending, true)); + + Assert.assertTrue(coordinator.isReusableApprovalGranted(mcpMetadata)); + Assert.assertFalse(coordinator.isReusableApprovalGranted( + Map.of("toolType", "MCP", "mcpId", "102"))); + Assert.assertFalse(coordinator.isReusableApprovalGranted( + Map.of("toolType", "MCP", "mcpName", "context7"))); + + coordinator.clearReusableApprovalScopes(); + + Assert.assertFalse(coordinator.isReusableApprovalGranted(mcpMetadata)); + } + + /** + * 验证受控 Shell 脚本只能按受信任内容摘要在当前 Turn 复用审批。 + */ + @Test + public void shouldReuseApprovedShellScriptScopeWithinCurrentTurn() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + Map scriptMetadata = Map.of( + "operateTool", true, + "operateToolType", "SHELL", + "approvalScope", "SHELL_SCRIPT:abc123"); + AgentPendingState pending = coordinator.register( + "session-1", "agent-1", "call-script", "execute_shell_command", "approve", + Map.of("command", "python3 report.py"), scriptMetadata, + Instant.now().plusSeconds(60), "batch-script"); + + coordinator.resolve(resume(pending, true)); + + Assert.assertTrue(coordinator.isReusableApprovalGranted(scriptMetadata)); + Assert.assertFalse(coordinator.isReusableApprovalGranted(Map.of( + "operateTool", true, + "operateToolType", "SHELL", + "approvalScope", "SHELL_SCRIPT:changed"))); + Assert.assertNull(coordinator.reusableApprovalScope(Map.of( + "approvalScope", "SHELL_SCRIPT:abc123"))); + } + + /** + * 验证跨节点受信任恢复可恢复 Shell 脚本内容摘要作用域。 + */ + @Test + public void shouldAuthorizeTrustedShellScriptScope() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + AgentResumeRequest request = new AgentResumeRequest(); + AgentResumeToken token = new AgentResumeToken(); + token.setValue("persisted-token"); + request.setResumeToken(token); + request.setApproved(true); + request.setTrusted(true); + request.setMetadata(Map.of( + "toolCallId", "call-script", + "toolName", "execute_shell_command", + "toolInput", Map.of("command", "node report.mjs"), + "operateTool", true, + "operateToolType", "SHELL", + "approvalScope", "SHELL_SCRIPT:def456")); + + coordinator.authorizeTrustedExecution(request); + + Assert.assertTrue(coordinator.isReusableApprovalGranted(Map.of( + "operateTool", true, + "operateToolType", "SHELL", + "approvalScope", "SHELL_SCRIPT:def456"))); + } + + /** + * 验证拒绝和过期不会产生可复用 MCP 批准。 + */ + @Test + public void shouldNotReuseRejectedOrExpiredMcpApproval() { + AgentToolApprovalCoordinator coordinator = AgentToolApprovalCoordinator.enabled(); + Map rejectedMetadata = Map.of("toolType", "MCP", "mcpId", "201"); + AgentPendingState rejected = coordinator.register( + "session-1", "agent-1", "call-rejected", "mcp_rejected", "approve", + Map.of(), rejectedMetadata, Instant.now().plusSeconds(60), "batch-rejected"); + coordinator.resolve(resume(rejected, false)); + + Map expiredMetadata = Map.of("toolType", "MCP", "mcpId", "202"); + AgentPendingState expired = coordinator.register( + "session-1", "agent-1", "call-expired-mcp", "mcp_expired", "approve", + Map.of(), expiredMetadata, Instant.now().minusSeconds(1), "batch-expired-mcp"); + coordinator.resolve(resume(expired, true)); + + Assert.assertFalse(coordinator.isReusableApprovalGranted(rejectedMetadata)); + Assert.assertFalse(coordinator.isReusableApprovalGranted(expiredMetadata)); + } + /** * 验证同一 toolCallId 不能被重新绑定到不同工具内容。 */ diff --git a/easy-agents-agui/pom.xml b/easy-agents-agui/pom.xml new file mode 100644 index 0000000..d6c44ce --- /dev/null +++ b/easy-agents-agui/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + + + com.easyagents + easy-agents + ${revision} + + + easy-agents-agui + easy-agents-agui + + + 17 + UTF-8 + + + + + com.easyagents + easy-agents-agent-runtime + + + io.agentscope + agentscope-extensions-agui + + + junit + junit + test + + + diff --git a/easy-agents-agui/src/main/java/com/easyagents/agui/AguiExtendedEvent.java b/easy-agents-agui/src/main/java/com/easyagents/agui/AguiExtendedEvent.java new file mode 100644 index 0000000..054b02d --- /dev/null +++ b/easy-agents-agui/src/main/java/com/easyagents/agui/AguiExtendedEvent.java @@ -0,0 +1,84 @@ +package com.easyagents.agui; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.agentscope.core.agui.model.AguiMessage; + +import java.util.List; +import java.util.Objects; + +/** + * AgentScope 1.0.12 尚未提供的 AG-UI 标准线级事件。 + * + *

该补充层只覆盖当前官方扩展缺失的标准事件,不复制 AG-UI 事件枚举。

+ */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = AguiExtendedEvent.RunError.class, name = "RUN_ERROR"), + @JsonSubTypes.Type(value = AguiExtendedEvent.MessagesSnapshot.class, name = "MESSAGES_SNAPSHOT") +}) +public sealed interface AguiExtendedEvent + permits AguiExtendedEvent.RunError, AguiExtendedEvent.MessagesSnapshot { + + /** + * 运行失败事件。 + * + * @param threadId AG-UI 线程 ID + * @param runId AG-UI 运行 ID + * @param message 可安全展示的错误消息 + * @param code 稳定错误码 + */ + record RunError(String threadId, String runId, String message, String code) + implements AguiExtendedEvent { + + /** + * 创建运行失败事件。 + * + * @param threadId AG-UI 线程 ID + * @param runId AG-UI 运行 ID + * @param message 可安全展示的错误消息 + * @param code 稳定错误码 + */ + @JsonCreator + public RunError( + @JsonProperty("threadId") String threadId, + @JsonProperty("runId") String runId, + @JsonProperty("message") String message, + @JsonProperty("code") String code) { + this.threadId = Objects.requireNonNull(threadId, "threadId cannot be null"); + this.runId = Objects.requireNonNull(runId, "runId cannot be null"); + this.message = Objects.requireNonNull(message, "message cannot be null"); + this.code = code; + } + } + + /** + * 消息全量快照事件。 + * + * @param threadId AG-UI 线程 ID + * @param runId AG-UI 运行 ID + * @param messages 消息快照 + */ + record MessagesSnapshot(String threadId, String runId, List messages) + implements AguiExtendedEvent { + + /** + * 创建消息全量快照事件。 + * + * @param threadId AG-UI 线程 ID + * @param runId AG-UI 运行 ID + * @param messages 消息快照 + */ + @JsonCreator + public MessagesSnapshot( + @JsonProperty("threadId") String threadId, + @JsonProperty("runId") String runId, + @JsonProperty("messages") List messages) { + this.threadId = Objects.requireNonNull(threadId, "threadId cannot be null"); + this.runId = Objects.requireNonNull(runId, "runId cannot be null"); + this.messages = messages == null ? List.of() : List.copyOf(messages); + } + } +} diff --git a/easy-agents-agui/src/main/java/com/easyagents/agui/AguiProtocolEventEncoder.java b/easy-agents-agui/src/main/java/com/easyagents/agui/AguiProtocolEventEncoder.java new file mode 100644 index 0000000..77173bc --- /dev/null +++ b/easy-agents-agui/src/main/java/com/easyagents/agui/AguiProtocolEventEncoder.java @@ -0,0 +1,39 @@ +package com.easyagents.agui; + +import io.agentscope.core.agui.AguiException; +import io.agentscope.core.util.JsonException; +import io.agentscope.core.util.JsonUtils; + +/** + * 将 AG-UI 事件编码为 JSON 或 SSE 数据帧。 + * + *

编码器无可变状态,可安全跨请求复用。

+ */ +public final class AguiProtocolEventEncoder { + + /** + * 编码为 JSON。 + * + * @param event AG-UI 官方事件或补充标准事件 + * @return JSON 文本 + * @throws AguiException.EncodingException 序列化失败时抛出 + */ + public String encodeToJson(Object event) { + try { + return JsonUtils.getJsonCodec().toJson(event); + } catch (JsonException exception) { + throw new AguiException.EncodingException("Failed to encode AG-UI event", exception); + } + } + + /** + * 编码为 SSE data 帧。 + * + * @param event AG-UI 官方事件或补充标准事件 + * @return 完整 SSE data 帧 + * @throws AguiException.EncodingException 序列化失败时抛出 + */ + public String encode(Object event) { + return "data: " + encodeToJson(event) + "\n\n"; + } +} diff --git a/easy-agents-agui/src/main/java/com/easyagents/agui/AguiRuntimeEventProjector.java b/easy-agents-agui/src/main/java/com/easyagents/agui/AguiRuntimeEventProjector.java new file mode 100644 index 0000000..4126b83 --- /dev/null +++ b/easy-agents-agui/src/main/java/com/easyagents/agui/AguiRuntimeEventProjector.java @@ -0,0 +1,249 @@ +package com.easyagents.agui; + +import com.easyagents.agent.runtime.event.AgentRuntimeEvent; +import com.easyagents.agent.runtime.event.AgentRuntimeEventType; +import io.agentscope.core.agui.event.AguiEvent; +import io.agentscope.core.util.JsonException; +import io.agentscope.core.util.JsonUtils; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * 将 Easy-Agents 中立运行时事件有序投影为 AG-UI 事件。 + * + *

实例绑定单个 run 且非线程安全。调用方应按运行顺序串行调用 {@link #project(AgentRuntimeEvent)}。

+ */ +public final class AguiRuntimeEventProjector { + + private static final String DEFAULT_ERROR_MESSAGE = "Agent runtime failed."; + + /** AG-UI thread ID。 */ + private final String threadId; + /** AG-UI run ID。 */ + private final String runId; + /** 已开始但尚未收到结果的工具调用。 */ + private final Set knownToolCallIds = new LinkedHashSet<>(); + + private boolean runStarted; + private boolean terminated; + private String openMessageId; + private String openReasoningMessageId; + private long generatedMessageSequence; + + /** + * 创建不含业务 Custom Event 的投影器。 + * + * @param threadId AG-UI thread ID + * @param runId AG-UI run ID + */ + public AguiRuntimeEventProjector(String threadId, String runId) { + this.threadId = requireText(threadId, "threadId"); + this.runId = requireText(runId, "runId"); + } + + /** + * 按输入顺序投影一条运行时事件。 + * + * @param event Easy-Agents 运行时事件 + * @return 零到多条 AG-UI 官方或补充标准事件 + */ + public List project(AgentRuntimeEvent event) { + if (event == null || event.getEventType() == null || terminated) { + return List.of(); + } + List output = new ArrayList<>(); + switch (event.getEventType()) { + case STARTED -> startRun(output); + case MESSAGE_DELTA -> projectMessageDelta(event, output); + case REASONING_STARTED -> startReasoning(event, output); + case REASONING_DELTA -> projectReasoningDelta(event, output); + case REASONING_COMPLETED -> closeReasoning(output); + case TOOL_CALL -> projectToolCall(event, output); + case TOOL_RESULT -> projectToolResult(event, output); + case COMPLETED -> finishSuccessfully(output); + case FAILED -> finishWithError(event, "AGENT_RUNTIME_FAILED", output); + case CANCELLED -> finishWithError(event, "RUN_CANCELLED", output); + default -> { + // EasyFlow 等上层业务扩展由各自协议边界映射为 CUSTOM,通用模块保持业务无关。 + } + } + return List.copyOf(output); + } + + /** + * 判断当前投影是否已经产生协议终态。 + * + * @return 已产生 RUN_FINISHED 或 RUN_ERROR 时为 true + */ + public boolean isTerminated() { + return terminated; + } + + private void startRun(List output) { + if (!runStarted) { + output.add(new AguiEvent.RunStarted(threadId, runId)); + runStarted = true; + } + } + + private void projectMessageDelta(AgentRuntimeEvent event, List output) { + startRun(output); + String messageId = eventMessageId(event, "assistant"); + if (!Objects.equals(openMessageId, messageId)) { + closeMessage(output); + output.add(new AguiEvent.TextMessageStart(threadId, runId, messageId, "assistant")); + openMessageId = messageId; + } + String delta = stringValue(event.getPayload(), "text"); + if (!delta.isEmpty()) { + output.add(new AguiEvent.TextMessageContent(threadId, runId, messageId, delta)); + } + } + + private void startReasoning(AgentRuntimeEvent event, List output) { + startRun(output); + if (openReasoningMessageId != null) { + return; + } + openReasoningMessageId = eventMessageId(event, "reasoning"); + output.add(new AguiEvent.ReasoningMessageStart( + threadId, runId, openReasoningMessageId, "reasoning")); + } + + private void projectReasoningDelta(AgentRuntimeEvent event, List output) { + startReasoning(event, output); + String delta = stringValue(event.getPayload(), "reasoning"); + if (!delta.isEmpty()) { + output.add(new AguiEvent.ReasoningMessageContent( + threadId, runId, openReasoningMessageId, delta)); + } + } + + private void projectToolCall(AgentRuntimeEvent event, List output) { + startRun(output); + String toolCallId = firstText(event.getToolCallId(), stringValue(event.getPayload(), "toolCallId")); + if (toolCallId == null || !knownToolCallIds.add(toolCallId)) { + return; + } + String toolName = firstText( + stringValue(event.getPayload(), "toolName"), + stringValue(event.getPayload(), "name"), + "tool"); + output.add(new AguiEvent.ToolCallStart(threadId, runId, toolCallId, toolName)); + output.add(new AguiEvent.ToolCallArgs( + threadId, runId, toolCallId, jsonValue(event.getPayload().get("input")))); + output.add(new AguiEvent.ToolCallEnd(threadId, runId, toolCallId)); + } + + private void projectToolResult(AgentRuntimeEvent event, List output) { + startRun(output); + String toolCallId = firstText(event.getToolCallId(), stringValue(event.getPayload(), "toolCallId")); + if (toolCallId == null || !knownToolCallIds.contains(toolCallId)) { + return; + } + String messageId = eventMessageId(event, "tool-" + toolCallId); + String content = nullToEmpty(firstText( + stringValue(event.getPayload(), "text"), + event.getPayload().containsKey("result") + ? jsonValue(event.getPayload().get("result")) + : null)); + output.add(new AguiEvent.ToolCallResult( + threadId, runId, toolCallId, content, "tool", messageId)); + } + + private void finishSuccessfully(List output) { + startRun(output); + closeOpenFragments(output); + output.add(new AguiEvent.RunFinished(threadId, runId)); + terminated = true; + } + + private void finishWithError(AgentRuntimeEvent event, String code, List output) { + startRun(output); + closeOpenFragments(output); + String message = firstText( + stringValue(event.getPayload(), "message"), + stringValue(event.getPayload(), "reason"), + DEFAULT_ERROR_MESSAGE); + output.add(new AguiExtendedEvent.RunError(threadId, runId, message, code)); + terminated = true; + } + + private void closeOpenFragments(List output) { + closeReasoning(output); + closeMessage(output); + } + + private void closeMessage(List output) { + if (openMessageId != null) { + output.add(new AguiEvent.TextMessageEnd(threadId, runId, openMessageId)); + openMessageId = null; + } + } + + private void closeReasoning(List output) { + if (openReasoningMessageId != null) { + output.add(new AguiEvent.ReasoningMessageEnd( + threadId, runId, openReasoningMessageId)); + openReasoningMessageId = null; + } + } + + private String eventMessageId(AgentRuntimeEvent event, String suffix) { + String messageId = firstText( + event.getMessageId(), + event.getMessage() == null ? null : event.getMessage().getMessageId()); + if (messageId != null) { + return messageId; + } + generatedMessageSequence++; + return runId + "-" + suffix + "-" + generatedMessageSequence; + } + + private static String stringValue(Map payload, String key) { + if (payload == null) { + return ""; + } + Object value = payload.get(key); + return value instanceof String text ? text : ""; + } + + private static String jsonValue(Object value) { + if (value == null) { + return "{}"; + } + if (value instanceof String text) { + return text; + } + try { + return JsonUtils.getJsonCodec().toJson(value); + } catch (JsonException exception) { + throw new IllegalArgumentException("Failed to encode AG-UI payload", exception); + } + } + + private static String firstText(String... values) { + for (String value : values) { + if (value != null && !value.isBlank()) { + return value; + } + } + return null; + } + + private static String requireText(String value, String name) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(name + " cannot be blank"); + } + return value; + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/easy-agents-agui/src/test/java/com/easyagents/agui/AguiProtocolEventEncoderTest.java b/easy-agents-agui/src/test/java/com/easyagents/agui/AguiProtocolEventEncoderTest.java new file mode 100644 index 0000000..4e9bf65 --- /dev/null +++ b/easy-agents-agui/src/test/java/com/easyagents/agui/AguiProtocolEventEncoderTest.java @@ -0,0 +1,43 @@ +package com.easyagents.agui; + +import io.agentscope.core.agui.event.AguiEvent; +import io.agentscope.core.agui.model.AguiMessage; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +/** + * {@link AguiProtocolEventEncoder} 的线级协议测试。 + */ +public class AguiProtocolEventEncoderTest { + + private final AguiProtocolEventEncoder encoder = new AguiProtocolEventEncoder(); + + /** + * 验证 AgentScope 官方事件保留 AG-UI type 字段。 + */ + @Test + public void shouldEncodeOfficialEvent() { + String json = encoder.encodeToJson(new AguiEvent.RunStarted("thread-1", "run-1")); + + Assert.assertTrue(json.contains("\"type\":\"RUN_STARTED\"")); + Assert.assertTrue(json.contains("\"threadId\":\"thread-1\"")); + } + + /** + * 验证补充的失败和消息快照事件使用现代 AG-UI 标准事件名。 + */ + @Test + public void shouldEncodeExtendedStandardEvents() { + String error = encoder.encodeToJson( + new AguiExtendedEvent.RunError("thread-1", "run-1", "boom", "FAILED")); + String snapshot = encoder.encodeToJson(new AguiExtendedEvent.MessagesSnapshot( + "thread-1", "run-1", List.of(AguiMessage.userMessage("message-1", "hello")))); + + Assert.assertTrue(error.contains("\"type\":\"RUN_ERROR\"")); + Assert.assertTrue(error.contains("\"code\":\"FAILED\"")); + Assert.assertTrue(snapshot.contains("\"type\":\"MESSAGES_SNAPSHOT\"")); + Assert.assertTrue(snapshot.contains("\"role\":\"user\"")); + } +} diff --git a/easy-agents-agui/src/test/java/com/easyagents/agui/AguiRuntimeEventProjectorTest.java b/easy-agents-agui/src/test/java/com/easyagents/agui/AguiRuntimeEventProjectorTest.java new file mode 100644 index 0000000..1b2ea3e --- /dev/null +++ b/easy-agents-agui/src/test/java/com/easyagents/agui/AguiRuntimeEventProjectorTest.java @@ -0,0 +1,90 @@ +package com.easyagents.agui; + +import com.easyagents.agent.runtime.event.AgentRuntimeEvent; +import com.easyagents.agent.runtime.event.AgentRuntimeEventType; +import io.agentscope.core.agui.event.AguiEvent; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * {@link AguiRuntimeEventProjector} 的协议顺序与终态测试。 + */ +public class AguiRuntimeEventProjectorTest { + + /** + * 验证文本、推理、工具和成功终态按 AG-UI 顺序投影。 + */ + @Test + public void shouldProjectSuccessfulRunInProtocolOrder() { + AguiRuntimeEventProjector projector = new AguiRuntimeEventProjector("thread-1", "run-1"); + List events = new ArrayList<>(); + + events.addAll(projector.project(event(AgentRuntimeEventType.STARTED, null, null, Map.of()))); + events.addAll(projector.project(event( + AgentRuntimeEventType.REASONING_STARTED, "reasoning-1", null, Map.of()))); + events.addAll(projector.project(event( + AgentRuntimeEventType.REASONING_DELTA, "reasoning-1", null, Map.of("reasoning", "分析")))); + events.addAll(projector.project(event( + AgentRuntimeEventType.REASONING_COMPLETED, "reasoning-1", null, Map.of()))); + events.addAll(projector.project(event( + AgentRuntimeEventType.MESSAGE_DELTA, "message-1", null, Map.of("text", "你好")))); + events.addAll(projector.project(event( + AgentRuntimeEventType.TOOL_CALL, null, "tool-1", + Map.of("toolName", "search", "input", Map.of("q", "AG-UI"))))); + events.addAll(projector.project(event( + AgentRuntimeEventType.TOOL_RESULT, null, "tool-1", Map.of("text", "done")))); + events.addAll(projector.project(event(AgentRuntimeEventType.COMPLETED, null, null, Map.of()))); + + Assert.assertEquals(List.of( + "RunStarted", + "ReasoningMessageStart", + "ReasoningMessageContent", + "ReasoningMessageEnd", + "TextMessageStart", + "TextMessageContent", + "ToolCallStart", + "ToolCallArgs", + "ToolCallEnd", + "ToolCallResult", + "TextMessageEnd", + "RunFinished"), events.stream().map(value -> value.getClass().getSimpleName()).toList()); + Assert.assertEquals( + "reasoning", ((AguiEvent.ReasoningMessageStart) events.get(1)).role()); + Assert.assertTrue(projector.isTerminated()); + } + + /** + * 验证失败终态不会追加成功事件,终态后的迟到事件会被丢弃。 + */ + @Test + public void shouldEmitSingleErrorTerminalAndIgnoreLateEvents() { + AguiRuntimeEventProjector projector = new AguiRuntimeEventProjector("thread-1", "run-1"); + + List failed = projector.project(event( + AgentRuntimeEventType.FAILED, null, null, Map.of("message", "boom"))); + List late = projector.project(event(AgentRuntimeEventType.COMPLETED, null, null, Map.of())); + + Assert.assertEquals(2, failed.size()); + Assert.assertTrue(failed.get(0) instanceof AguiEvent.RunStarted); + Assert.assertEquals( + new AguiExtendedEvent.RunError("thread-1", "run-1", "boom", "AGENT_RUNTIME_FAILED"), + failed.get(1)); + Assert.assertTrue(late.isEmpty()); + } + + private static AgentRuntimeEvent event( + AgentRuntimeEventType type, + String messageId, + String toolCallId, + Map payload) { + AgentRuntimeEvent event = AgentRuntimeEvent.of(type); + event.setMessageId(messageId); + event.setToolCallId(toolCallId); + event.setPayload(payload); + return event; + } +} diff --git a/easy-agents-bom/pom.xml b/easy-agents-bom/pom.xml index 32234e4..254b793 100644 --- a/easy-agents-bom/pom.xml +++ b/easy-agents-bom/pom.xml @@ -264,6 +264,10 @@ com.easyagents easy-agents-agent-runtime + + com.easyagents + easy-agents-agui + diff --git a/pom.xml b/pom.xml index 455abe1..69a6bbb 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ easy-agents-mcp easy-agents-skill easy-agents-agent-runtime + easy-agents-agui easy-agents-flow easy-agents-support @@ -456,12 +457,24 @@ ${revision} + + com.easyagents + easy-agents-agui + ${revision} + + io.agentscope agentscope ${agentscope.version} + + io.agentscope + agentscope-extensions-agui + ${agentscope.version} + + From c7d410d7559847f7e182b1d29e7d3e69bd513b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 19 Aug 2026 21:51:16 +0800 Subject: [PATCH 29/33] =?UTF-8?q?feat:=20=E5=AE=8C=E5=96=84=20Agent=20Skil?= =?UTF-8?q?l=20=E6=B8=90=E8=BF=9B=E6=8A=AB=E9=9C=B2=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持 Skill 绑定 MCP 冻结清单和延迟注册 - 拒绝同步工具工作流进入不可恢复挂起状态 --- .../agentscope/AgentScopeReActRuntime.java | 33 +- .../agentscope/AgentScopeSkillAdapter.java | 56 ++- .../agentscope/AgentScopeToolAdapter.java | 1 + .../observer/SkillExecutionObserver.java | 3 + .../event/observer/ToolExecutionObserver.java | 40 +-- .../runtime/mcp/FrozenMcpClientWrapper.java | 155 ++++++++ .../agent/runtime/mcp/McpRegistration.java | 26 ++ .../runtime/mcp/McpSkillRegistration.java | 88 +++++ .../easyagents/agent/runtime/mcp/McpSpec.java | 59 ++++ .../agent/runtime/mcp/McpToolManifest.java | 332 ++++++++++++++++++ .../runtime/mcp/McpToolManifestEntry.java | 105 ++++++ .../agent/runtime/mcp/McpToolkitAdapter.java | 79 ++++- .../runtime/skill/AgentSkillBinding.java | 29 +- .../runtime/skill/AgentSkillLoadCall.java | 34 ++ .../skill/AgentSkillRuntimeContext.java | 11 +- .../runtime/mcp/McpToolManifestTest.java | 168 +++++++++ .../runtime/mcp/McpToolkitAdapterTest.java | 137 ++++++++ .../core/chain/runtime/ChainExecutor.java | 51 ++- .../test/ChainExecutorConcurrencyTest.java | 69 ++++ 19 files changed, 1417 insertions(+), 59 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/FrozenMcpClientWrapper.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSkillRegistration.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifest.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifestEntry.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolManifestTest.java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java index b59265b..a013812 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeReActRuntime.java @@ -18,6 +18,7 @@ import com.easyagents.agent.runtime.knowledge.citation.AgentKnowledgeCitationMat import com.easyagents.agent.runtime.knowledge.citation.HeuristicKnowledgeCitationMatcher; import com.easyagents.agent.runtime.message.*; import com.easyagents.agent.runtime.mcp.McpRegistration; +import com.easyagents.agent.runtime.mcp.McpSkillRegistration; import com.easyagents.agent.runtime.mcp.McpSpecValidator; import com.easyagents.agent.runtime.mcp.McpToolkitAdapter; import com.easyagents.agent.runtime.persistence.session.noop.NoopAgentSessionStore; @@ -218,7 +219,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { saveSession(); return Flux.just(started(executionContext), cancelled(executionContext)); }).doOnNext(event -> executionContext.getConversationRecorder().record(executionContext, event)) - .doFinally(signalType -> cleanupTurn()); + .doFinally(signalType -> cleanupStreamSegment(false)); } return runAgentStreamAfterLock(executionContext, List::of); }); @@ -238,6 +239,8 @@ public class AgentScopeReActRuntime implements AgentRuntime { if (!running.compareAndSet(false, true)) { return Flux.error(new AgentRuntimeException("Agent runtime is already streaming.")); } + // 新用户消息建立新的 Turn,上一 Turn 的 MCP 级批准不能跨轮复用。 + approvalCoordinator.clearReusableApprovalScopes(); return runAgentStreamAfterLock(executionContext, inputSupplier); }); } @@ -296,8 +299,8 @@ public class AgentScopeReActRuntime implements AgentRuntime { .doOnNext(event -> executionContext.getConversationRecorder().record(executionContext, event)) // 处理中断请求 .doOnCancel(() -> cancelInternal(executionContext, sideEvents, finalText, finalMessage, cancelled)) - // 释放运行锁并清掉 turn context。 - .doFinally(signalType -> cleanupTurn()); + // HITL 挂起时保留当前 Turn 的 MCP 批准,其余终态完整清理。 + .doFinally(signalType -> cleanupStreamSegment(suspendedEvent.get() != null)); } /** @@ -582,7 +585,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { suspended.getMetadata().put("approvalStatus", AgentToolApprovalResolution.Status.WAITING.name()); return Flux.just(started(context), suspended) .doOnNext(event -> context.getConversationRecorder().record(context, event)) - .doFinally(signalType -> cleanupTurn()); + .doFinally(signalType -> cleanupStreamSegment(true)); } /** @@ -791,10 +794,15 @@ public class AgentScopeReActRuntime implements AgentRuntime { } /** - * 清理本轮状态。 + * 清理一次 stream/resume 片段状态。 + * + * @param preserveReusableApprovalScopes 是否因 HITL 挂起而保留当前 Turn 的 MCP 批准 */ - private void cleanupTurn() { + private void cleanupStreamSegment(boolean preserveReusableApprovalScopes) { approvalCoordinator.clearExecutionAuthorizations(); + if (!preserveReusableApprovalScopes) { + approvalCoordinator.clearReusableApprovalScopes(); + } turnContextHolder.clear(); running.set(false); } @@ -1123,7 +1131,8 @@ public class AgentScopeReActRuntime implements AgentRuntime { AgentScopeMemoryBuildResult memoryResult = memoryAdapter.createMemoryResult(null, definition.getMemoryPolicy(), model); Memory memory = memoryResult.getMemory(); Knowledge knowledge = knowledgeAdapter.createAggregateKnowledge(context, turnContextHolder); - SkillBox skillBox = skillAdapter.createSkillBox(definition.getSkillBoxSpec(), toolkit, skillTools); + SkillBox skillBox = skillAdapter.createSkillBox(definition.getSkillBoxSpec(), toolkit, skillTools, + toolkitBuildResult.skillMcpRegistrations()); // AutoContextInterceptor 是官方 AutoContextHook 的替代实现。这里仍只注册统一 runtime hook, // 避免官方 hook 与 Easy-Agents interceptor 同时触发压缩和 inputMessages 改写。 AgentRuntimeEventBridge eventBridge = new AgentRuntimeEventBridge(context, turnContextHolder); @@ -1202,7 +1211,7 @@ public class AgentScopeReActRuntime implements AgentRuntime { Toolkit toolkit) { Map> skillTools = new LinkedHashMap<>(); if (!context.getAgentDefinition().getExecutionOptions().isToolCallingEnabled()) { - return new AgentScopeToolkitBuildResult(skillTools, List.of(), List.of()); + return new AgentScopeToolkitBuildResult(skillTools, List.of(), List.of(), List.of()); } for (AgentToolSpec toolSpec : context.getAgentDefinition().getToolSpecs()) { AgentToolInvoker invoker = context.getToolInvokers().get(toolSpec.getName()); @@ -1222,7 +1231,8 @@ public class AgentScopeReActRuntime implements AgentRuntime { context.getAgentDefinition().getOperateToolSpecs(), toolkit); McpSpecValidator.validateToolConflicts(context.getAgentDefinition().getToolSpecs(), mcpRegistration.getToolSpecs(), context.getAgentDefinition().getOperateToolSpecs()); - return new AgentScopeToolkitBuildResult(skillTools, mcpRegistration.getToolSpecs(), operateToolSpecs); + return new AgentScopeToolkitBuildResult(skillTools, mcpRegistration.getToolSpecs(), operateToolSpecs, + mcpRegistration.getSkillRegistrations()); } private List mergeToolSpecs(List toolSpecs, @@ -1290,7 +1300,8 @@ public class AgentScopeReActRuntime implements AgentRuntime { } private record AgentScopeToolkitBuildResult(Map> skillTools, - List mcpToolSpecs, - List operateToolSpecs) { + List mcpToolSpecs, + List operateToolSpecs, + List skillMcpRegistrations) { } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeSkillAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeSkillAdapter.java index 0a2e9bb..3be5391 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeSkillAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeSkillAdapter.java @@ -4,12 +4,14 @@ import com.easyagents.agent.runtime.AgentRuntimeException; import com.easyagents.agent.runtime.skill.AgentSkillBoxSpec; import com.easyagents.agent.runtime.skill.AgentSkillCompiler; import com.easyagents.agent.runtime.skill.AgentSkillSpec; +import com.easyagents.agent.runtime.mcp.McpSkillRegistration; import io.agentscope.core.skill.AgentSkill; import io.agentscope.core.skill.SkillBox; import io.agentscope.core.tool.AgentTool; import io.agentscope.core.tool.Toolkit; import java.util.List; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -67,6 +69,22 @@ public class AgentScopeSkillAdapter implements AgentSkillCompiler { * @return SkillBox;未配置 Skill 时返回 null */ public SkillBox createSkillBox(AgentSkillBoxSpec spec, Toolkit toolkit, Map> skillTools) { + return createSkillBox(spec, toolkit, skillTools, List.of()); + } + + /** + * 创建并绑定静态工具及 MCP 工具的 AgentScope SkillBox。 + * + * @param spec SkillBox 声明 + * @param toolkit Toolkit 实例 + * @param skillTools 按 Skill ID 分组的静态工具 + * @param skillMcpRegistrations 按 Skill 延迟激活的 MCP client + * @return SkillBox;未配置 Skill 时返回 null + */ + public SkillBox createSkillBox(AgentSkillBoxSpec spec, + Toolkit toolkit, + Map> skillTools, + List skillMcpRegistrations) { if (spec == null || spec.getSkills().isEmpty()) { return null; } @@ -74,10 +92,15 @@ public class AgentScopeSkillAdapter implements AgentSkillCompiler { ? new SkillBox(toolkit) : new SkillBox(toolkit, spec.getSkillBoxId()); skillBox.setExposeAllSkillMetadata(spec.isExposeAllSkillMetadata()); + Map> mcpBySkill = groupMcpRegistrations(skillMcpRegistrations); for (AgentSkillSpec skillSpec : spec.getSkills()) { AgentSkill skill = compile(skillSpec); List tools = skillTools == null ? List.of() : skillTools.getOrDefault(skillSpec.getSkillId(), List.of()); - if (tools.isEmpty()) { + List mcpRegistrations = mcpBySkill.remove(skillSpec.getSkillId()); + if (mcpRegistrations == null) { + mcpRegistrations = List.of(); + } + if (tools.isEmpty() && mcpRegistrations.isEmpty()) { skillBox.registration() .skill(skill) .toolkit(toolkit) @@ -97,11 +120,42 @@ public class AgentScopeSkillAdapter implements AgentSkillCompiler { .agentTool(tool) .apply(); } + for (McpSkillRegistration mcpRegistration : mcpRegistrations) { + skillBox.registration() + .skill(skill) + .toolkit(toolkit) + .enableTools(mcpRegistration.getEnableTools()) + .disableTools(mcpRegistration.getDisableTools()) + .presetParameters(mcpRegistration.getPresetParameters()) + .mcpClient(mcpRegistration.getClient()) + .apply(); + } + } + if (!mcpBySkill.isEmpty()) { + throw new AgentRuntimeException("Skill-bound MCP references unknown skill: " + + mcpBySkill.keySet().iterator().next()); } skillBox.syncToolGroupStates(); return skillBox; } + private Map> groupMcpRegistrations( + List registrations) { + Map> grouped = new LinkedHashMap<>(); + if (registrations == null) { + return grouped; + } + for (McpSkillRegistration registration : registrations) { + if (registration == null || registration.getSkillId() == null + || registration.getSkillId().isBlank()) { + throw new AgentRuntimeException("Skill-bound MCP skill id is required."); + } + grouped.computeIfAbsent(registration.getSkillId(), key -> new java.util.ArrayList<>()) + .add(registration); + } + return grouped; + } + /** * 校验 Skill 声明是否具备 AgentScope 注册和模型提示所需的必要信息。 * diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeToolAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeToolAdapter.java index ccf460b..ccc236d 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeToolAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/agentscope/AgentScopeToolAdapter.java @@ -561,6 +561,7 @@ public class AgentScopeToolAdapter { } target.put("skillId", binding.getSkillId()); target.put("skillName", binding.getSkillName()); + target.put("skillDisplayName", binding.getSkillDisplayName()); target.put("skillBoxId", binding.getSkillBoxId()); } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/SkillExecutionObserver.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/SkillExecutionObserver.java index a2e9740..58b3281 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/SkillExecutionObserver.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/SkillExecutionObserver.java @@ -270,10 +270,12 @@ public class SkillExecutionObserver implements AgentRuntimeObserver { } event.getPayload().put("skillId", call.getSkillId()); event.getPayload().put("skillName", call.getSkillName()); + event.getPayload().put("skillDisplayName", call.getSkillDisplayName()); event.getPayload().put("skillBoxId", call.getSkillBoxId()); event.getPayload().put("path", call.getPath()); event.getMetadata().put("skillId", call.getSkillId()); event.getMetadata().put("skillName", call.getSkillName()); + event.getMetadata().put("skillDisplayName", call.getSkillDisplayName()); event.getMetadata().put("skillBoxId", call.getSkillBoxId()); } @@ -283,6 +285,7 @@ public class SkillExecutionObserver implements AgentRuntimeObserver { } target.put("skillId", binding.getSkillId()); target.put("skillName", binding.getSkillName()); + target.put("skillDisplayName", binding.getSkillDisplayName()); target.put("skillBoxId", binding.getSkillBoxId()); } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/ToolExecutionObserver.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/ToolExecutionObserver.java index 14b39c7..0f8f9af 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/ToolExecutionObserver.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/event/observer/ToolExecutionObserver.java @@ -9,8 +9,6 @@ import com.easyagents.agent.runtime.tool.AgentToolSpec; import io.agentscope.core.hook.HookEvent; import io.agentscope.core.hook.PostActingEvent; import io.agentscope.core.hook.PreActingEvent; -import io.agentscope.core.message.ContentBlock; -import io.agentscope.core.message.TextBlock; import io.agentscope.core.message.ToolResultBlock; import io.agentscope.core.message.ToolUseBlock; import reactor.core.publisher.Mono; @@ -103,12 +101,7 @@ public class ToolExecutionObserver implements AgentRuntimeObserver { runtimeEvent.getPayload().put("toolCallId", toolUse.getId()); runtimeEvent.getPayload().put("name", toolUse.getName()); runtimeEvent.getPayload().put("toolName", toolUse.getName()); - runtimeEvent.getPayload().put("input", toolUse.getInput()); - runtimeEvent.getPayload().put("content", toolUse.getContent()); runtimeEvent.getPayload().put("status", "RUNNING"); - runtimeEvent.getPayload().put("source", "HOOK"); - runtimeEvent.getPayload().put("phase", "PRE_ACTING"); - runtimeEvent.getMetadata().putAll(nullToEmpty(toolUse.getMetadata())); enrichToolPayload(runtimeEvent, toolUse.getName()); eventBridge.emit(runtimeEvent); } @@ -129,15 +122,8 @@ public class ToolExecutionObserver implements AgentRuntimeObserver { runtimeEvent.getPayload().put("toolCallId", toolCallId); runtimeEvent.getPayload().put("name", toolName); runtimeEvent.getPayload().put("toolName", toolName); - runtimeEvent.getPayload().put("text", resultText(result)); - runtimeEvent.getPayload().put("suspended", result != null && result.isSuspended()); runtimeEvent.getPayload().put("status", success(result) ? "SUCCESS" : "FAILED"); runtimeEvent.getPayload().put("success", success(result)); - runtimeEvent.getPayload().put("source", "HOOK"); - runtimeEvent.getPayload().put("phase", "POST_ACTING"); - if (result != null) { - runtimeEvent.getMetadata().putAll(nullToEmpty(result.getMetadata())); - } enrichToolPayload(runtimeEvent, toolName); eventBridge.emit(runtimeEvent); } @@ -149,12 +135,7 @@ public class ToolExecutionObserver implements AgentRuntimeObserver { } Map metadata = toolSpec.getMetadata(); putIfPresent(runtimeEvent.getPayload(), metadata, "toolDisplayName"); - putIfPresent(runtimeEvent.getPayload(), metadata, "rawMcpToolName"); - putIfPresent(runtimeEvent.getPayload(), metadata, "mcpToolName"); - putIfPresent(runtimeEvent.getPayload(), metadata, "mcpName"); - putIfPresent(runtimeEvent.getPayload(), metadata, "mcpTitle"); - putIfPresent(runtimeEvent.getPayload(), metadata, "source"); - runtimeEvent.getMetadata().putAll(metadata); + putIfPresent(runtimeEvent.getPayload(), metadata, "skillId"); } private void putIfPresent(Map payload, Map metadata, String key) { @@ -171,25 +152,6 @@ public class ToolExecutionObserver implements AgentRuntimeObserver { return !(success instanceof Boolean) || Boolean.TRUE.equals(success); } - private String resultText(ToolResultBlock result) { - if (result == null || result.getOutput() == null || result.getOutput().isEmpty()) { - return ""; - } - StringBuilder builder = new StringBuilder(); - for (ContentBlock block : result.getOutput()) { - if (block instanceof TextBlock textBlock) { - builder.append(textBlock.getText()); - } else { - builder.append(block); - } - } - return builder.toString(); - } - - private Map nullToEmpty(Map map) { - return map == null ? new LinkedHashMap<>() : map; - } - private boolean isSkillTool(String toolName) { if (skillContext == null) { return false; diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/FrozenMcpClientWrapper.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/FrozenMcpClientWrapper.java new file mode 100644 index 0000000..6dad341 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/FrozenMcpClientWrapper.java @@ -0,0 +1,155 @@ +package com.easyagents.agent.runtime.mcp; + +import io.agentscope.core.tool.mcp.McpClientWrapper; +import io.modelcontextprotocol.spec.McpSchema; +import reactor.core.publisher.Mono; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 将一次验证通过的远端 MCP Tool 清单冻结为只读白名单视图。 + */ +final class FrozenMcpClientWrapper extends McpClientWrapper { + + private final McpClientWrapper delegate; + private final List frozenTools; + private final Map runtimeToRaw = new LinkedHashMap<>(); + + /** + * 创建冻结 MCP client 视图。 + * + * @param delegate 原始 client + * @param actualTools 已一次性读取并验证的远端 Tool + * @param manifest 冻结清单 + * @param aliases 显式运行别名 + * @param prefix 运行名前缀 + */ + FrozenMcpClientWrapper(McpClientWrapper delegate, + List actualTools, + List manifest, + Map aliases, + String prefix) { + super(delegate == null ? "mcp" : delegate.getName()); + this.delegate = delegate; + this.frozenTools = freeze(actualTools, manifest, aliases, prefix); + this.frozenTools.forEach(tool -> cachedTools.put(tool.name(), tool)); + } + + /** {@inheritDoc} */ + @Override + public Mono initialize() { + return delegate.initialize().doOnSuccess(ignored -> initialized = delegate.isInitialized()); + } + + /** {@inheritDoc} */ + @Override + public Mono> listTools() { + return Mono.just(frozenTools); + } + + /** {@inheritDoc} */ + @Override + public Mono callTool(String toolName, Map arguments) { + return delegate.callTool(runtimeToRaw.getOrDefault(toolName, toolName), arguments); + } + + /** {@inheritDoc} */ + @Override + public void close() { + delegate.close(); + initialized = false; + } + + /** + * 按冻结 manifest 顺序裁剪并应用稳定运行别名。 + * + * @param actualTools 远端当前 Tool + * @param manifest 冻结清单 + * @param aliases 显式别名 + * @param prefix 动态前缀 + * @return 不可变 Tool 白名单 + */ + private List freeze(List actualTools, + List manifest, + Map aliases, + String prefix) { + Map actualByName = new LinkedHashMap<>(); + if (actualTools != null) { + actualTools.stream().filter(tool -> tool != null && tool.name() != null) + .forEach(tool -> actualByName.put(tool.name(), tool)); + } + Map usedRuntimeNames = new LinkedHashMap<>(); + List result = new ArrayList<>(); + for (McpToolManifestEntry entry : manifest) { + McpSchema.Tool actual = actualByName.get(entry.getName()); + if (actual == null) { + throw new IllegalStateException("Frozen MCP tool is missing after validation: " + entry.getName()); + } + String runtimeName = uniqueRuntimeName( + runtimeName(actual.name(), aliases, prefix), actual.name(), usedRuntimeNames); + runtimeToRaw.put(runtimeName, actual.name()); + Map meta = new LinkedHashMap<>(); + if (actual.meta() != null) { + meta.putAll(actual.meta()); + } + if (!runtimeName.equals(actual.name())) { + meta.put(AliasedMcpClientWrapper.RAW_TOOL_NAME_META_KEY, actual.name()); + } + // 模型可见描述也必须来自发布时冻结清单,避免远端描述在运行中漂移。 + result.add(new McpSchema.Tool(runtimeName, actual.title(), entry.getDescription(), + actual.inputSchema(), actual.outputSchema(), actual.annotations(), meta)); + } + return List.copyOf(result); + } + + /** + * 计算单个 Tool 的运行名。 + * + * @param rawName 原始名称 + * @param aliases 显式别名 + * @param prefix 动态前缀 + * @return 运行名 + */ + private String runtimeName(String rawName, Map aliases, String prefix) { + String alias = aliases == null ? null : aliases.get(rawName); + if (alias != null && !alias.isBlank()) { + return alias; + } + if (prefix == null || prefix.isBlank()) { + return rawName; + } + String segment = String.valueOf(rawName == null ? "" : rawName).trim() + .replaceAll("[^A-Za-z0-9_-]", "_") + .replaceAll("_+", "_"); + return prefix.trim() + (segment.isBlank() ? "tool" : segment); + } + + /** + * 避免别名碰撞。 + * + * @param candidate 候选运行名 + * @param rawName 原始名称 + * @param used 已使用运行名 + * @return 唯一运行名 + */ + private String uniqueRuntimeName(String candidate, + String rawName, + Map used) { + String existing = used.get(candidate); + if (existing == null || existing.equals(rawName)) { + used.put(candidate, rawName); + return candidate; + } + int suffix = 2; + String value = candidate + "_" + suffix; + while (used.containsKey(value)) { + suffix++; + value = candidate + "_" + suffix; + } + used.put(value, rawName); + return value; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpRegistration.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpRegistration.java index 2550406..32952c8 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpRegistration.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpRegistration.java @@ -13,6 +13,7 @@ public class McpRegistration { private final List clients; private final List toolSpecs; + private final List skillRegistrations; /** * 创建 MCP 注册结果。 @@ -21,8 +22,24 @@ public class McpRegistration { * @param toolSpecs 已注册工具声明 */ public McpRegistration(List clients, List toolSpecs) { + this(clients, toolSpecs, List.of()); + } + + /** + * 创建 MCP 注册结果。 + * + * @param clients 已创建 MCP client + * @param toolSpecs 已发现工具声明 + * @param skillRegistrations 等待注册到 Skill 的 MCP client + */ + public McpRegistration(List clients, + List toolSpecs, + List skillRegistrations) { this.clients = clients == null ? List.of() : new ArrayList<>(clients); this.toolSpecs = toolSpecs == null ? List.of() : new ArrayList<>(toolSpecs); + this.skillRegistrations = skillRegistrations == null + ? List.of() + : new ArrayList<>(skillRegistrations); } /** @@ -51,4 +68,13 @@ public class McpRegistration { public List getToolSpecs() { return toolSpecs; } + + /** + * 获取等待注册到 Skill 的 MCP client。 + * + * @return Skill MCP 注册声明 + */ + public List getSkillRegistrations() { + return skillRegistrations; + } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSkillRegistration.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSkillRegistration.java new file mode 100644 index 0000000..63bf01d --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSkillRegistration.java @@ -0,0 +1,88 @@ +package com.easyagents.agent.runtime.mcp; + +import io.agentscope.core.tool.mcp.McpClientWrapper; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 等待注册到指定 Skill 的 MCP client。 + */ +public class McpSkillRegistration { + + private final String skillId; + private final McpClientWrapper client; + private final List enableTools; + private final List disableTools; + private final Map> presetParameters; + + /** + * 创建 Skill MCP 注册声明。 + * + * @param skillId Skill ID + * @param client MCP client + * @param enableTools 运行时工具白名单 + * @param disableTools 运行时工具黑名单 + * @param presetParameters 预设参数 + */ + public McpSkillRegistration(String skillId, + McpClientWrapper client, + List enableTools, + List disableTools, + Map> presetParameters) { + this.skillId = skillId; + this.client = client; + this.enableTools = enableTools == null ? List.of() : new ArrayList<>(enableTools); + this.disableTools = disableTools == null ? List.of() : new ArrayList<>(disableTools); + this.presetParameters = presetParameters == null + ? Map.of() + : new LinkedHashMap<>(presetParameters); + } + + /** + * 获取 Skill ID。 + * + * @return Skill ID + */ + public String getSkillId() { + return skillId; + } + + /** + * 获取 MCP client。 + * + * @return MCP client + */ + public McpClientWrapper getClient() { + return client; + } + + /** + * 获取运行时工具白名单。 + * + * @return 工具白名单 + */ + public List getEnableTools() { + return enableTools; + } + + /** + * 获取运行时工具黑名单。 + * + * @return 工具黑名单 + */ + public List getDisableTools() { + return disableTools; + } + + /** + * 获取预设参数。 + * + * @return 预设参数 + */ + public Map> getPresetParameters() { + return presetParameters; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSpec.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSpec.java index 8ebb80b..c585898 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSpec.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpSpec.java @@ -33,6 +33,9 @@ public class McpSpec { private boolean approvalRequired; private AgentToolApprovalRequest approvalRequest = new AgentToolApprovalRequest(); private Map toolApprovalRequests = new LinkedHashMap<>(); + private String skillId; + private List frozenToolManifest = new ArrayList<>(); + private String frozenToolManifestHash; private Map metadata = new LinkedHashMap<>(); /** @@ -406,6 +409,62 @@ public class McpSpec { : new LinkedHashMap<>(toolApprovalRequests); } + /** + * 获取所属 Skill ID。 + * + * @return Skill ID;未绑定 Skill 时为空 + */ + public String getSkillId() { + return skillId; + } + + /** + * 设置所属 Skill ID。 + * + * @param skillId Skill ID + */ + public void setSkillId(String skillId) { + this.skillId = skillId; + } + + /** + * 获取冻结 Tool 清单。 + * + * @return 冻结 Tool 清单 + */ + public List getFrozenToolManifest() { + return frozenToolManifest; + } + + /** + * 设置冻结 Tool 清单。 + * + * @param frozenToolManifest 冻结 Tool 清单 + */ + public void setFrozenToolManifest(List frozenToolManifest) { + this.frozenToolManifest = frozenToolManifest == null + ? new ArrayList<>() + : new ArrayList<>(frozenToolManifest); + } + + /** + * 获取冻结 Tool 清单 hash。 + * + * @return 清单 hash + */ + public String getFrozenToolManifestHash() { + return frozenToolManifestHash; + } + + /** + * 设置冻结 Tool 清单 hash。 + * + * @param frozenToolManifestHash 清单 hash + */ + public void setFrozenToolManifestHash(String frozenToolManifestHash) { + this.frozenToolManifestHash = frozenToolManifestHash; + } + /** * 获取元数据。 * diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifest.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifest.java new file mode 100644 index 0000000..ac8fa4d --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifest.java @@ -0,0 +1,332 @@ +package com.easyagents.agent.runtime.mcp; + +import com.alibaba.fastjson2.JSON; +import com.easyagents.agent.runtime.AgentRuntimeException; +import io.modelcontextprotocol.spec.McpSchema; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * MCP Tool 冻结清单规范化与完整性校验器。 + */ +public final class McpToolManifest { + + /** MCP Tool 原始名称允许的最大 Unicode 字符数。 */ + public static final int MAX_TOOL_NAME_LENGTH = 128; + + /** MCP Tool 描述允许的最大 Unicode 字符数。 */ + public static final int MAX_TOOL_DESCRIPTION_LENGTH = 4_096; + + /** 单个输入或输出 Schema 允许的最大 UTF-8 字节数。 */ + public static final int MAX_SCHEMA_UTF8_BYTES = 256 * 1_024; + + /** 完整规范化 Manifest 允许的最大 UTF-8 字节数。 */ + public static final int MAX_MANIFEST_UTF8_BYTES = 2 * 1_024 * 1_024; + + private McpToolManifest() { + } + + /** + * 将 MCP Tool 转换为稳定清单项。 + * + * @param tools MCP Tool 列表 + * @return 按名称稳定排序的清单 + */ + public static List fromTools(List tools) { + if (tools == null || tools.isEmpty()) { + return List.of(); + } + List entries = new ArrayList<>(); + Set names = new HashSet<>(); + int manifestBytes = 2; + for (McpSchema.Tool tool : tools) { + if (tool == null || tool.name() == null || tool.name().isBlank()) { + continue; + } + if (!names.add(tool.name())) { + throw new AgentRuntimeException("Duplicate MCP tool name: " + tool.name()); + } + McpToolManifestEntry entry = new McpToolManifestEntry(); + entry.setName(tool.name()); + entry.setDescription(normalizeText(tool.description())); + entry.setInputSchema(normalizeSchema("MCP tool input schema", tool.inputSchema())); + entry.setOutputSchema(normalizeSchema("MCP tool output schema", tool.outputSchema())); + assertEntryBounds(entry); + manifestBytes += JSON.toJSONString(toCanonicalValue(entry)) + .getBytes(StandardCharsets.UTF_8).length; + if (!entries.isEmpty()) { + manifestBytes++; + } + if (manifestBytes > MAX_MANIFEST_UTF8_BYTES) { + throw new AgentRuntimeException("MCP tool manifest exceeds " + + MAX_MANIFEST_UTF8_BYTES + " UTF-8 bytes."); + } + entries.add(entry); + } + entries.sort(Comparator.comparing(McpToolManifestEntry::getName)); + assertManifestSize(entries); + return List.copyOf(entries); + } + + /** + * 计算冻结清单的 SHA-256。 + * + * @param entries 冻结清单 + * @return 十六进制 SHA-256 + */ + public static String hash(List entries) { + String json = canonicalJson(entries); + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(json.getBytes(StandardCharsets.UTF_8)); + return java.util.HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException error) { + throw new AgentRuntimeException("SHA-256 is unavailable for MCP manifest validation.", error); + } + } + + /** + * 校验远端 Tool 与冻结白名单一致,同时忽略远端新增 Tool。 + * + * @param spec MCP 运行时声明 + * @param actualTools 远端当前 Tool + * @throws AgentRuntimeException 冻结清单缺失、被篡改、Tool 缺失或 Schema 漂移时抛出 + */ + public static void assertFrozenManifest(McpSpec spec, List actualTools) { + if (spec == null || spec.getSkillId() == null || spec.getSkillId().isBlank()) { + return; + } + List expected = spec.getFrozenToolManifest(); + String expectedHash = spec.getFrozenToolManifestHash(); + if (expected == null || expected.isEmpty() || expectedHash == null || expectedHash.isBlank()) { + throw new AgentRuntimeException("Skill-bound MCP requires a frozen tool manifest: " + spec.getName()); + } + if (!expectedHash.equals(hash(expected))) { + throw new AgentRuntimeException("Skill-bound MCP frozen tool manifest is invalid: " + spec.getName()); + } + Set frozenNames = new HashSet<>(); + for (McpToolManifestEntry entry : expected) { + if (entry != null && entry.getName() != null && !entry.getName().isBlank()) { + frozenNames.add(entry.getName()); + } + } + Map actualByName = new LinkedHashMap<>(); + if (actualTools != null) { + for (McpSchema.Tool tool : actualTools) { + if (tool == null || tool.name() == null || !frozenNames.contains(tool.name())) { + continue; + } + if (actualByName.containsKey(tool.name())) { + throw new AgentRuntimeException("Duplicate MCP tool name: " + tool.name()); + } + List normalized = fromTools(List.of(tool)); + if (!normalized.isEmpty()) { + actualByName.put(tool.name(), normalized.get(0)); + } + } + } + for (McpToolManifestEntry expectedEntry : expected) { + if (expectedEntry == null || expectedEntry.getName() == null || expectedEntry.getName().isBlank()) { + throw new AgentRuntimeException("Skill-bound MCP frozen tool name is required: " + spec.getName()); + } + McpToolManifestEntry actualEntry = actualByName.get(expectedEntry.getName()); + if (actualEntry == null) { + throw new AgentRuntimeException("Skill-bound MCP tool is missing: " + expectedEntry.getName()); + } + if (!sameRuntimeSchema(expectedEntry, actualEntry)) { + throw new AgentRuntimeException("Skill-bound MCP tool schema has changed: " + + expectedEntry.getName()); + } + } + } + + /** + * 比较 Runtime 必须锁定的 Tool 名称及输入、输出 Schema。 + * + *

描述用于保存与发布阶段的完整 manifest 变更识别,但远端仅调整描述时不会改变 + * 已发布 Tool 的可调用边界,因此运行时不应中断既有 Agent。

+ * + * @param expected 冻结清单项 + * @param actual 远端当前清单项 + * @return 名称及 Schema 相同时返回 {@code true} + */ + private static boolean sameRuntimeSchema(McpToolManifestEntry expected, + McpToolManifestEntry actual) { + return java.util.Objects.equals(expected.getName(), actual.getName()) + && java.util.Objects.equals(normalizeJson(expected.getInputSchema()), + normalizeJson(actual.getInputSchema())) + && java.util.Objects.equals(normalizeJson(expected.getOutputSchema()), + normalizeJson(actual.getOutputSchema())); + } + + /** + * 将冻结清单转换为稳定 JSON,并对反序列化后的清单执行同等边界校验。 + * + * @param entries 冻结清单 + * @return 稳定 JSON + * @throws AgentRuntimeException 清单包含重复名称或超出预算时抛出 + */ + private static String canonicalJson(List entries) { + List> canonical = new ArrayList<>(); + Set names = new HashSet<>(); + if (entries != null) { + entries.stream() + .filter(entry -> entry != null && entry.getName() != null && !entry.getName().isBlank()) + .sorted(Comparator.comparing(McpToolManifestEntry::getName)) + .forEach(entry -> { + if (!names.add(entry.getName())) { + throw new AgentRuntimeException("Duplicate MCP tool name: " + entry.getName()); + } + McpToolManifestEntry normalized = new McpToolManifestEntry(); + normalized.setName(entry.getName()); + normalized.setDescription(normalizeText(entry.getDescription())); + normalized.setInputSchema(normalizeSchema( + "MCP tool input schema", entry.getInputSchema())); + normalized.setOutputSchema(normalizeSchema( + "MCP tool output schema", entry.getOutputSchema())); + assertEntryBounds(normalized); + canonical.add(toCanonicalValue(normalized)); + }); + } + String json = JSON.toJSONString(canonical); + assertUtf8Size("MCP tool manifest", json, MAX_MANIFEST_UTF8_BYTES); + return json; + } + + /** + * 校验单个清单项的名称、描述及 Schema 预算。 + * + * @param entry 已规范化的清单项 + * @throws AgentRuntimeException 任一字段超出预算时抛出 + */ + private static void assertEntryBounds(McpToolManifestEntry entry) { + assertTextLength("MCP tool name", entry.getName(), MAX_TOOL_NAME_LENGTH); + assertTextLength("MCP tool description", entry.getDescription(), MAX_TOOL_DESCRIPTION_LENGTH); + assertSchemaSize("MCP tool input schema", entry.getInputSchema()); + assertSchemaSize("MCP tool output schema", entry.getOutputSchema()); + } + + /** + * 校验规范化清单的聚合字节预算。 + * + * @param entries 已规范化且排序的清单 + * @throws AgentRuntimeException 清单超出聚合预算时抛出 + */ + private static void assertManifestSize(List entries) { + List> canonical = entries.stream() + .map(McpToolManifest::toCanonicalValue) + .toList(); + assertUtf8Size("MCP tool manifest", JSON.toJSONString(canonical), MAX_MANIFEST_UTF8_BYTES); + } + + /** + * 构造用于哈希和预算计算的稳定清单值。 + * + * @param entry 已规范化的清单项 + * @return 保持字段顺序的清单值 + */ + private static Map toCanonicalValue(McpToolManifestEntry entry) { + Map value = new LinkedHashMap<>(); + value.put("name", entry.getName()); + value.put("description", normalizeText(entry.getDescription())); + value.put("inputSchema", entry.getInputSchema()); + value.put("outputSchema", entry.getOutputSchema()); + return value; + } + + /** + * 校验 Unicode 字符长度,避免 UTF-16 代理对被重复计数。 + * + * @param field 字段名称 + * @param value 字段值 + * @param maxLength 最大 Unicode 字符数 + * @throws AgentRuntimeException 字段超长时抛出 + */ + private static void assertTextLength(String field, String value, int maxLength) { + if (value != null && value.codePointCount(0, value.length()) > maxLength) { + throw new AgentRuntimeException(field + " exceeds " + maxLength + " characters."); + } + } + + /** + * 校验单个 Schema 的 UTF-8 字节预算。 + * + * @param field Schema 字段名称 + * @param schema 已规范化 Schema + * @throws AgentRuntimeException Schema 超出预算时抛出 + */ + private static void assertSchemaSize(String field, Object schema) { + if (schema != null) { + assertUtf8Size(field, JSON.toJSONString(schema), MAX_SCHEMA_UTF8_BYTES); + } + } + + /** + * 校验 JSON 或文本的 UTF-8 字节长度。 + * + * @param field 字段名称 + * @param value 待校验文本 + * @param maxBytes 最大 UTF-8 字节数 + * @throws AgentRuntimeException 文本超出预算时抛出 + */ + private static void assertUtf8Size(String field, String value, int maxBytes) { + int bytes = value.getBytes(StandardCharsets.UTF_8).length; + if (bytes > maxBytes) { + throw new AgentRuntimeException(field + " exceeds " + maxBytes + " UTF-8 bytes."); + } + } + + /** + * 在解析和排序前限制原始 Schema,避免超大输入进入规范化流程。 + * + * @param field Schema 字段名称 + * @param value 原始 Schema + * @return 规范化 Schema + * @throws AgentRuntimeException 原始 Schema 超出预算时抛出 + */ + private static Object normalizeSchema(String field, Object value) { + if (value == null) { + return null; + } + String json = JSON.toJSONString(value); + assertUtf8Size(field, json, MAX_SCHEMA_UTF8_BYTES); + return sortJson(JSON.parse(json)); + } + + private static Object normalizeJson(Object value) { + if (value == null) { + return null; + } + return sortJson(JSON.parse(JSON.toJSONString(value))); + } + + private static Object sortJson(Object value) { + if (value instanceof Map source) { + Map sorted = new TreeMap<>(); + source.forEach((key, child) -> sorted.put(String.valueOf(key), sortJson(child))); + return sorted; + } + if (value instanceof List source) { + List sorted = new ArrayList<>(source.size()); + for (Object child : source) { + sorted.add(sortJson(child)); + } + return sorted; + } + return value; + } + + private static String normalizeText(String value) { + return value == null ? "" : value; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifestEntry.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifestEntry.java new file mode 100644 index 0000000..6eb16bd --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolManifestEntry.java @@ -0,0 +1,105 @@ +package com.easyagents.agent.runtime.mcp; + +import java.util.Objects; + +/** + * MCP Tool 冻结清单项。 + */ +public class McpToolManifestEntry { + + private String name; + private String description; + private Object inputSchema; + private Object outputSchema; + + /** + * 获取 Tool 名称。 + * + * @return Tool 名称 + */ + public String getName() { + return name; + } + + /** + * 设置 Tool 名称。 + * + * @param name Tool 名称 + */ + public void setName(String name) { + this.name = name; + } + + /** + * 获取 Tool 描述。 + * + * @return Tool 描述 + */ + public String getDescription() { + return description; + } + + /** + * 设置 Tool 描述。 + * + * @param description Tool 描述 + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * 获取输入 Schema。 + * + * @return 输入 Schema + */ + public Object getInputSchema() { + return inputSchema; + } + + /** + * 设置输入 Schema。 + * + * @param inputSchema 输入 Schema + */ + public void setInputSchema(Object inputSchema) { + this.inputSchema = inputSchema; + } + + /** + * 获取输出 Schema。 + * + * @return 输出 Schema + */ + public Object getOutputSchema() { + return outputSchema; + } + + /** + * 设置输出 Schema。 + * + * @param outputSchema 输出 Schema + */ + public void setOutputSchema(Object outputSchema) { + this.outputSchema = outputSchema; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof McpToolManifestEntry that)) { + return false; + } + return Objects.equals(name, that.name) + && Objects.equals(description, that.description) + && Objects.equals(inputSchema, that.inputSchema) + && Objects.equals(outputSchema, that.outputSchema); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, inputSchema, outputSchema); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapter.java index f82c0b9..ab1adb7 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapter.java @@ -52,6 +52,7 @@ public class McpToolkitAdapter { } List clients = new ArrayList<>(); List toolSpecs = new ArrayList<>(); + List skillRegistrations = new ArrayList<>(); try { for (McpSpec spec : specs) { if (spec == null) { @@ -59,16 +60,59 @@ public class McpToolkitAdapter { } McpSpecValidator.validateConnection(spec); McpClientWrapper client = clientFactory.create(spec); - client = applyAliases(spec, client); clients.add(client); - registerClient(spec, client, toolkit); + if (isSkillBound(spec)) { + List actualTools = initializeAndListTools(client); + McpToolManifest.assertFrozenManifest(spec, actualTools); + client = new FrozenMcpClientWrapper(client, actualTools, + spec.getFrozenToolManifest(), spec.getToolAliases(), spec.getToolNamePrefix()); + } else { + client = applyAliases(spec, client); + } + clients.set(clients.size() - 1, client); + if (isSkillBound(spec)) { + // Skill MCP 必须以冻结 manifest 派生白名单,调用方不能通过空列表放宽到远端全部 Tool。 + spec.setEnableTools(frozenRuntimeToolNames(spec, client)); + skillRegistrations.add(new McpSkillRegistration( + spec.getSkillId(), client, spec.getEnableTools(), spec.getDisableTools(), + spec.getPresetParameters())); + } else { + registerClient(spec, client, toolkit); + } toolSpecs.addAll(toToolSpecs(spec, registeredTools(spec, client))); } } catch (RuntimeException error) { closeQuietly(clients); throw error; } - return new McpRegistration(clients, toolSpecs); + return new McpRegistration(clients, toolSpecs, skillRegistrations); + } + + /** + * 根据冻结原始 Tool 名称和别名后的远端清单生成强制运行白名单。 + * + * @param spec Skill MCP 声明 + * @param client 已应用运行别名的 client + * @return 冻结 Tool 对应的运行名 + */ + private List frozenRuntimeToolNames(McpSpec spec, McpClientWrapper client) { + Set frozenRawNames = new LinkedHashSet<>(); + for (McpToolManifestEntry entry : spec.getFrozenToolManifest()) { + if (entry != null && entry.getName() != null && !entry.getName().isBlank()) { + frozenRawNames.add(entry.getName()); + } + } + List names = new ArrayList<>(); + for (McpSchema.Tool tool : listTools(client)) { + if (tool != null && frozenRawNames.contains(rawToolName(spec, tool))) { + names.add(tool.name()); + } + } + if (names.size() != frozenRawNames.size()) { + throw new AgentRuntimeException("Skill-bound MCP frozen tool aliases are incomplete: " + + spec.getName()); + } + return List.copyOf(names); } private McpClientWrapper applyAliases(McpSpec spec, McpClientWrapper client) { @@ -95,7 +139,7 @@ public class McpToolkitAdapter { } private List registeredTools(McpSpec spec, McpClientWrapper client) { - List tools = client.listTools().block(); + List tools = listTools(client); if (tools == null || tools.isEmpty()) { return List.of(); } @@ -108,6 +152,30 @@ public class McpToolkitAdapter { return filtered; } + private List listTools(McpClientWrapper client) { + List tools = client.listTools().block(); + return tools == null ? List.of() : tools; + } + + /** + * 初始化 client 后读取一次远端 Tool 清单。 + * + * @param client MCP client + * @return Tool 清单 + */ + private List initializeAndListTools(McpClientWrapper client) { + // AgentScope validates the initialized flag when listTools() is invoked. Build the + // second publisher only after initialization has completed, otherwise eager publisher + // assembly can fail even though the server initializes successfully moments later. + client.initialize().block(); + List tools = client.listTools().block(); + return tools == null ? List.of() : tools; + } + + private boolean isSkillBound(McpSpec spec) { + return spec.getSkillId() != null && !spec.getSkillId().isBlank(); + } + private boolean shouldRegister(String toolName, List enableTools, List disableTools) { if (enableTools != null && !enableTools.isEmpty()) { return enableTools.contains(toolName); @@ -166,6 +234,9 @@ public class McpToolkitAdapter { metadata.put("rawMcpToolName", rawToolName(spec, tool)); metadata.put("toolDisplayName", toolDisplayName(spec, tool)); metadata.put("transportType", spec.getTransportType().configValue()); + if (isSkillBound(spec)) { + metadata.put("skillId", spec.getSkillId()); + } return metadata; } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillBinding.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillBinding.java index f009fb0..eee5af8 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillBinding.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillBinding.java @@ -7,6 +7,7 @@ public class AgentSkillBinding { private final String skillId; private final String skillName; + private final String skillDisplayName; private final String skillBoxId; /** @@ -17,8 +18,26 @@ public class AgentSkillBinding { * @param skillBoxId SkillBox ID */ public AgentSkillBinding(String skillId, String skillName, String skillBoxId) { + this(skillId, skillName, skillName, skillBoxId); + } + + /** + * 创建带展示名称的 Skill 绑定关系。 + * + * @param skillId Skill ID + * @param skillName Skill 规范名称 + * @param skillDisplayName Skill 展示名称 + * @param skillBoxId SkillBox ID + */ + public AgentSkillBinding(String skillId, + String skillName, + String skillDisplayName, + String skillBoxId) { this.skillId = skillId; this.skillName = skillName; + this.skillDisplayName = skillDisplayName == null || skillDisplayName.isBlank() + ? skillName + : skillDisplayName; this.skillBoxId = skillBoxId; } @@ -40,6 +59,15 @@ public class AgentSkillBinding { return skillName; } + /** + * 获取 Skill 展示名称。 + * + * @return Skill 展示名称 + */ + public String getSkillDisplayName() { + return skillDisplayName; + } + /** * 获取 SkillBox ID。 * @@ -49,4 +77,3 @@ public class AgentSkillBinding { return skillBoxId; } } - diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillLoadCall.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillLoadCall.java index 5684a1a..f87636e 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillLoadCall.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillLoadCall.java @@ -11,6 +11,7 @@ public class AgentSkillLoadCall { private final String toolCallId; private final String skillId; private final String skillName; + private final String skillDisplayName; private final String skillBoxId; private final String path; private final Map input; @@ -31,9 +32,33 @@ public class AgentSkillLoadCall { String skillBoxId, String path, Map input) { + this(toolCallId, skillId, skillName, skillName, skillBoxId, path, input); + } + + /** + * 创建带展示名称的 Skill 加载工具调用记录。 + * + * @param toolCallId 工具调用 ID + * @param skillId Skill ID + * @param skillName Skill 规范名称 + * @param skillDisplayName Skill 展示名称 + * @param skillBoxId SkillBox ID + * @param path 资源路径 + * @param input 工具输入 + */ + public AgentSkillLoadCall(String toolCallId, + String skillId, + String skillName, + String skillDisplayName, + String skillBoxId, + String path, + Map input) { this.toolCallId = toolCallId; this.skillId = skillId; this.skillName = skillName; + this.skillDisplayName = skillDisplayName == null || skillDisplayName.isBlank() + ? skillName + : skillDisplayName; this.skillBoxId = skillBoxId; this.path = path; this.input = input == null ? new LinkedHashMap<>() : new LinkedHashMap<>(input); @@ -66,6 +91,15 @@ public class AgentSkillLoadCall { return skillName; } + /** + * 获取 Skill 展示名称。 + * + * @return Skill 展示名称 + */ + public String getSkillDisplayName() { + return skillDisplayName; + } + /** * 获取 SkillBox ID。 * diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillRuntimeContext.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillRuntimeContext.java index 9e76667..7aace60 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillRuntimeContext.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/skill/AgentSkillRuntimeContext.java @@ -43,7 +43,8 @@ public class AgentSkillRuntimeContext { continue; } skillBindings.put(skillSpec.getSkillId(), - new AgentSkillBinding(skillSpec.getSkillId(), skillSpec.getName(), spec.getSkillBoxId())); + new AgentSkillBinding(skillSpec.getSkillId(), skillSpec.getName(), + displayName(skillSpec), spec.getSkillBoxId())); } Map toolBindings = new LinkedHashMap<>(); for (Map.Entry> entry : spec.getToolBindings().entrySet()) { @@ -171,6 +172,7 @@ public class AgentSkillRuntimeContext { AgentSkillBinding binding = getSkillBinding(skillId); AgentSkillLoadCall call = new AgentSkillLoadCall(toolCallId, skillId, binding == null ? null : binding.getSkillName(), + binding == null ? null : binding.getSkillDisplayName(), binding == null ? null : binding.getSkillBoxId(), path, input); pendingLoadCalls.put(toolCallId, call); return call; @@ -206,4 +208,11 @@ public class AgentSkillRuntimeContext { private static String stringValue(Object value) { return value == null ? null : String.valueOf(value); } + + private static String displayName(AgentSkillSpec skillSpec) { + Object value = skillSpec.getMetadata() == null ? null : skillSpec.getMetadata().get("displayName"); + return value == null || String.valueOf(value).isBlank() + ? skillSpec.getName() + : String.valueOf(value); + } } diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolManifestTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolManifestTest.java new file mode 100644 index 0000000..70d7eda --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolManifestTest.java @@ -0,0 +1,168 @@ +package com.easyagents.agent.runtime.mcp; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import io.modelcontextprotocol.spec.McpSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * 测试 MCP Tool 冻结清单的稳定化与服务端预算。 + */ +public class McpToolManifestTest { + + /** + * 验证远端返回重复原始 Tool 名称时立即拒绝。 + */ + @Test + public void shouldRejectDuplicateRawToolNames() { + expectManifestFailure( + () -> McpToolManifest.fromTools(List.of(tool("search", "first", smallSchema()), + tool("search", "second", smallSchema()))), + "Duplicate"); + } + + /** + * 验证 Tool 名称超出字符预算时拒绝。 + */ + @Test + public void shouldRejectOverlongToolName() { + String name = "n".repeat(McpToolManifest.MAX_TOOL_NAME_LENGTH + 1); + + expectManifestFailure(() -> McpToolManifest.fromTools(List.of( + tool(name, "description", smallSchema()))), "name"); + } + + /** + * 验证 Tool 描述超出字符预算时拒绝。 + */ + @Test + public void shouldRejectOverlongToolDescription() { + String description = "d".repeat(McpToolManifest.MAX_TOOL_DESCRIPTION_LENGTH + 1); + + expectManifestFailure(() -> McpToolManifest.fromTools(List.of( + tool("search", description, smallSchema()))), "description"); + } + + /** + * 验证单个输入或输出 Schema 超出 UTF-8 预算时拒绝。 + */ + @Test + public void shouldRejectOversizedSingleSchema() { + McpSchema.JsonSchema oversized = schemaWithDescription( + "x".repeat(McpToolManifest.MAX_SCHEMA_UTF8_BYTES)); + + expectManifestFailure(() -> McpToolManifest.fromTools(List.of( + tool("search", "description", oversized))), "schema"); + } + + /** + * 验证各 Schema 合法但规范化 Manifest 聚合超过预算时拒绝。 + */ + @Test + public void shouldRejectOversizedAggregateManifest() { + McpSchema.JsonSchema schema = schemaWithDescription("x".repeat(220_000)); + List tools = new ArrayList<>(); + for (int index = 0; index < 10; index++) { + tools.add(new McpSchema.Tool("tool_" + index, "tool_" + index, "description", + schema, null, null, null)); + } + + expectManifestFailure(() -> McpToolManifest.fromTools(tools), "manifest"); + } + + /** + * 验证哈希入口同样拒绝反序列化后的重复名称,避免绕过发布阶段校验。 + */ + @Test + public void shouldRejectDuplicateNamesWhenHashingFrozenManifest() { + McpToolManifestEntry first = manifestEntry("search"); + McpToolManifestEntry second = manifestEntry("search"); + + expectManifestFailure(() -> McpToolManifest.hash(List.of(first, second)), "Duplicate"); + } + + /** + * 验证运行时完全忽略冻结白名单外新增 Tool,即使新增 Tool 超出发布清单预算。 + */ + @Test + public void shouldIgnoreOversizedRemoteToolOutsideFrozenWhitelist() { + McpSchema.Tool frozenTool = tool("search", "description", smallSchema()); + McpSpec spec = new McpSpec(); + spec.setName("demo"); + spec.setSkillId("skill-1"); + spec.setFrozenToolManifest(McpToolManifest.fromTools(List.of(frozenTool))); + spec.setFrozenToolManifestHash(McpToolManifest.hash(spec.getFrozenToolManifest())); + McpSchema.Tool extraTool = tool( + "new_remote_tool", + "x".repeat(McpToolManifest.MAX_TOOL_DESCRIPTION_LENGTH + 1), + smallSchema()); + + McpToolManifest.assertFrozenManifest(spec, List.of(frozenTool, extraTool)); + } + + /** + * 构造普通 MCP Tool。 + * + * @param name Tool 名称 + * @param description Tool 描述 + * @param schema 输入 Schema + * @return MCP Tool + */ + private McpSchema.Tool tool(String name, String description, McpSchema.JsonSchema schema) { + return new McpSchema.Tool(name, name, description, schema, null, null, null); + } + + /** + * 构造小型合法 Schema。 + * + * @return 合法 Schema + */ + private McpSchema.JsonSchema smallSchema() { + return schemaWithDescription("query"); + } + + /** + * 构造带指定属性描述的 Schema。 + * + * @param description 属性描述 + * @return MCP JSON Schema + */ + private McpSchema.JsonSchema schemaWithDescription(String description) { + return new McpSchema.JsonSchema("object", + Map.of("value", Map.of("type", "string", "description", description)), + List.of("value"), null, null, null); + } + + /** + * 构造最小冻结清单项。 + * + * @param name Tool 名称 + * @return 冻结清单项 + */ + private McpToolManifestEntry manifestEntry(String name) { + McpToolManifestEntry entry = new McpToolManifestEntry(); + entry.setName(name); + entry.setDescription("description"); + entry.setInputSchema(Map.of("type", "object")); + return entry; + } + + /** + * 断言清单转换抛出包含指定片段的运行时异常。 + * + * @param action 待执行动作 + * @param messageFragment 预期错误片段 + */ + private void expectManifestFailure(Runnable action, String messageFragment) { + try { + action.run(); + Assert.fail("Expected MCP manifest validation failure."); + } catch (AgentRuntimeException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains(messageFragment)); + } + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapterTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapterTest.java index 345fd50..31be0a5 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapterTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/mcp/McpToolkitAdapterTest.java @@ -1,6 +1,9 @@ package com.easyagents.agent.runtime.mcp; import com.easyagents.agent.runtime.AgentRuntimeException; +import com.easyagents.agent.runtime.agentscope.AgentScopeSkillAdapter; +import com.easyagents.agent.runtime.skill.AgentSkillBoxSpec; +import com.easyagents.agent.runtime.skill.AgentSkillSpec; import com.easyagents.agent.runtime.tool.AgentToolSpec; import io.agentscope.core.message.ToolResultBlock; import io.agentscope.core.tool.Toolkit; @@ -161,6 +164,130 @@ public class McpToolkitAdapterTest { } } + /** + * 验证 Skill MCP 只注册到禁用的 Skill Tool Group,加载前不向模型暴露。 + */ + @Test + public void shouldRegisterSkillMcpAsInactiveSkillToolGroup() { + List frozenTools = List.of(tool("search")); + FakeMcpClientWrapper client = new FakeMcpClientWrapper("demo", + List.of(tool("search"), tool("new_remote_tool"))); + McpToolkitAdapter adapter = new McpToolkitAdapter(new FakeMcpClientFactory(client)); + McpSpec spec = stdioSpec(); + spec.setSkillId("skill-1"); + spec.setToolAliases(Map.of("search", "skill_1_mcp_search")); + spec.setFrozenToolManifest(McpToolManifest.fromTools(frozenTools)); + spec.setFrozenToolManifestHash(McpToolManifest.hash(spec.getFrozenToolManifest())); + Toolkit toolkit = new Toolkit(); + + McpRegistration registration = adapter.register(List.of(spec), toolkit); + + Assert.assertNull(toolkit.getTool("skill_1_mcp_search")); + Assert.assertEquals(1, registration.getSkillRegistrations().size()); + Assert.assertEquals(List.of("skill_1_mcp_search"), + registration.getSkillRegistrations().get(0).getEnableTools()); + AgentSkillSpec skill = new AgentSkillSpec(); + skill.setSkillId("skill-1"); + skill.setName("Search Skill"); + skill.setDescription("Search through MCP."); + skill.setSkillContent("Load this skill before searching."); + AgentSkillBoxSpec skillBoxSpec = new AgentSkillBoxSpec(); + skillBoxSpec.setSkills(List.of(skill)); + + new AgentScopeSkillAdapter().createSkillBox(skillBoxSpec, toolkit, Map.of(), + registration.getSkillRegistrations()); + + Assert.assertNotNull(toolkit.getTool("skill_1_mcp_search")); + Assert.assertFalse(toolkit.getActiveGroups().contains("skill-1_skill_tools")); + Assert.assertTrue(toolkit.getToolSchemas().stream() + .noneMatch(schema -> "skill_1_mcp_search".equals(schema.getName()))); + Assert.assertNull(toolkit.getTool("skill_1_mcp_new_remote_tool")); + Assert.assertEquals(1, client.remoteListCalls.get()); + } + + /** + * 验证 Skill MCP 会在读取 Tool 清单前完成异步初始化。 + */ + @Test + public void shouldInitializeSkillMcpBeforeListingTools() { + FakeMcpClientWrapper client = new FakeMcpClientWrapper("demo", List.of(tool("search"))); + client.deferInitialization = true; + McpToolkitAdapter adapter = new McpToolkitAdapter(new FakeMcpClientFactory(client)); + McpSpec spec = stdioSpec(); + spec.setSkillId("skill-1"); + spec.setFrozenToolManifest(McpToolManifest.fromTools(List.of(tool("search")))); + spec.setFrozenToolManifestHash(McpToolManifest.hash(spec.getFrozenToolManifest())); + + McpRegistration registration = adapter.register(List.of(spec), new Toolkit()); + + Assert.assertTrue(client.isInitialized()); + Assert.assertEquals(1, client.remoteListCalls.get()); + Assert.assertEquals(1, registration.getSkillRegistrations().size()); + } + + /** + * 验证冻结 Tool 缺失时拒绝注册并关闭 client。 + */ + @Test + public void shouldRejectMissingFrozenSkillMcpToolAndCloseClient() { + FakeMcpClientWrapper client = new FakeMcpClientWrapper("demo", List.of(tool("other"))); + McpToolkitAdapter adapter = new McpToolkitAdapter(new FakeMcpClientFactory(client)); + McpSpec spec = stdioSpec(); + spec.setSkillId("skill-1"); + spec.setFrozenToolManifest(McpToolManifest.fromTools(List.of(tool("search")))); + spec.setFrozenToolManifestHash(McpToolManifest.hash(spec.getFrozenToolManifest())); + + try { + adapter.register(List.of(spec), new Toolkit()); + Assert.fail("Expected frozen MCP tool validation failure."); + } catch (AgentRuntimeException expected) { + Assert.assertTrue(expected.getMessage().contains("missing")); + Assert.assertTrue(client.closed.get()); + } + } + + /** + * 验证冻结 Tool Schema 漂移时拒绝注册。 + */ + @Test(expected = AgentRuntimeException.class) + public void shouldRejectChangedFrozenSkillMcpSchema() { + McpSchema.Tool expectedTool = tool("search"); + McpSchema.JsonSchema changedSchema = new McpSchema.JsonSchema("object", + Map.of("keyword", Map.of("type", "string")), List.of("keyword"), null, null, null); + McpSchema.Tool actualTool = new McpSchema.Tool("search", "search", "search description", + changedSchema, null, null, null); + McpToolkitAdapter adapter = new McpToolkitAdapter( + new FakeMcpClientFactory(new FakeMcpClientWrapper("demo", List.of(actualTool)))); + McpSpec spec = stdioSpec(); + spec.setSkillId("skill-1"); + spec.setFrozenToolManifest(McpToolManifest.fromTools(List.of(expectedTool))); + spec.setFrozenToolManifestHash(McpToolManifest.hash(spec.getFrozenToolManifest())); + + adapter.register(List.of(spec), new Toolkit()); + } + + /** + * 验证远端仅调整 Tool 描述时不破坏已发布 Skill 的运行兼容性。 + */ + @Test + public void shouldAllowChangedDescriptionWhenFrozenSchemaIsStable() { + McpSchema.Tool expectedTool = tool("search"); + McpSchema.Tool actualTool = new McpSchema.Tool( + "search", "search", "updated description", + expectedTool.inputSchema(), expectedTool.outputSchema(), null, null); + McpToolkitAdapter adapter = new McpToolkitAdapter( + new FakeMcpClientFactory(new FakeMcpClientWrapper("demo", List.of(actualTool)))); + McpSpec spec = stdioSpec(); + spec.setSkillId("skill-1"); + spec.setFrozenToolManifest(McpToolManifest.fromTools(List.of(expectedTool))); + spec.setFrozenToolManifestHash(McpToolManifest.hash(spec.getFrozenToolManifest())); + + McpRegistration registration = adapter.register(List.of(spec), new Toolkit()); + + Assert.assertEquals(1, registration.getSkillRegistrations().size()); + Assert.assertEquals("search description", registration.getToolSpecs().get(0).getDescription()); + } + private McpSpec stdioSpec() { McpSpec spec = new McpSpec(); spec.setName("demo"); @@ -197,7 +324,10 @@ public class McpToolkitAdapterTest { private final List tools; private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicReference lastCalledToolName = new AtomicReference<>(); + private final java.util.concurrent.atomic.AtomicInteger remoteListCalls = + new java.util.concurrent.atomic.AtomicInteger(); private boolean failOnListTools; + private boolean deferInitialization; private FakeMcpClientWrapper(String name, List tools) { super(name); @@ -206,12 +336,19 @@ public class McpToolkitAdapterTest { @Override public Mono initialize() { + if (deferInitialization) { + return Mono.fromRunnable(() -> initialized = true); + } initialized = true; return Mono.empty(); } @Override public Mono> listTools() { + if (!initialized) { + return Mono.error(new IllegalStateException("client is not initialized")); + } + remoteListCalls.incrementAndGet(); if (failOnListTools) { return Mono.error(new IllegalStateException("list tools failed")); } diff --git a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java index a752ada..ba130f3 100644 --- a/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java +++ b/easy-agents-flow/src/main/java/com/easyagents/flow/core/chain/runtime/ChainExecutor.java @@ -194,18 +194,44 @@ public class ChainExecutor { public Map execute(String definitionId, Map variables) { - return execute(definitionId, variables, Long.MAX_VALUE, TimeUnit.SECONDS); + return executeInternal(definitionId, variables, Long.MAX_VALUE, TimeUnit.SECONDS, false); } public Map execute(String definitionId, Map variables, long timeout, TimeUnit unit) { + return executeInternal(definitionId, variables, timeout, unit, false); + } + + /** + * 同步执行不允许进入人工挂起状态的工作流。 + * + *

该入口适用于 Tool 等无法把工作流恢复协议接回原调用方的同步场景。 + * 工作流一旦进入 {@link ChainStatus#SUSPEND},实例会被取消并立即返回失败。

+ * + * @param definitionId 工作流定义 ID + * @param variables 输入变量 + * @return 工作流输出 + * @throws RuntimeException 工作流失败、挂起或执行线程被中断时抛出 + */ + public Map executeWithoutSuspension( + String definitionId, Map variables) { + return executeInternal( + definitionId, variables, Long.MAX_VALUE, TimeUnit.SECONDS, true); + } + + private Map executeInternal( + String definitionId, + Map variables, + long timeout, + TimeUnit unit, + boolean rejectSuspension) { Chain chain = createChain(definitionId); String stateInstanceId = chain.getStateInstanceId(); try { chain.start(variables); Map result = awaitPersistentOutcome( - stateInstanceId, timeout, unit, null); + stateInstanceId, timeout, unit, null, rejectSuspension); clearDefaultStates(result); return result; } catch (TimeoutException e) { @@ -759,6 +785,17 @@ public class ChainExecutor { TimeUnit unit, Chain parentChain) throws InterruptedException, TimeoutException { + return awaitPersistentOutcome( + stateInstanceId, timeout, unit, parentChain, false); + } + + private Map awaitPersistentOutcome( + String stateInstanceId, + long timeout, + TimeUnit unit, + Chain parentChain, + boolean rejectSuspension) + throws InterruptedException, TimeoutException { Objects.requireNonNull(unit, "time unit required"); long timeoutNanos = timeout == Long.MAX_VALUE ? Long.MAX_VALUE @@ -789,6 +826,12 @@ public class ChainExecutor { "Chain state not found: " + stateInstanceId); } ChainStatus status = state.getStatus(); + if (rejectSuspension && status == ChainStatus.SUSPEND) { + cancel(stateInstanceId, "Suspended workflow is not supported by this caller"); + throw new ChainException( + "Workflow suspended and requires external input: " + + stateInstanceId); + } if (status != null && status.isTerminal()) { if (!status.isSuccess()) { ExceptionSummary error = state.getError(); @@ -1087,6 +1130,10 @@ public class ChainExecutor { // 状态已过期或被清理时,该触发器已经失去业务目标,直接确认避免无限热重放。 return; } + if (state.getStatus() != null && state.getStatus().isTerminal()) { + // 终态不可再次执行;直接确认迟到或重复触发器,避免重新加载已清理的定义快照。 + return; + } ChainDefinition definition = getDefinitionForInstance(state); diff --git a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java index b9d402d..266147d 100644 --- a/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java +++ b/easy-agents-flow/src/test/java/com/easyagents/flow/core/test/ChainExecutorConcurrencyTest.java @@ -33,6 +33,7 @@ import com.easyagents.flow.core.chain.runtime.Trigger; import com.easyagents.flow.core.chain.runtime.TriggerScheduler; import com.easyagents.flow.core.node.EndNode; import com.easyagents.flow.core.node.BaseNode; +import com.easyagents.flow.core.node.ConfirmNode; import com.easyagents.flow.core.node.StartNode; import org.junit.Assert; import org.junit.Test; @@ -46,6 +47,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -59,6 +61,43 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ChainExecutorConcurrencyTest { + /** + * 验证同步 Tool 入口遇到人工挂起会快速失败,不会无限占用调用线程。 + * + * @throws Exception 异步测试执行失败时抛出 + */ + @Test + public void shouldFailFastWhenNonSuspendingExecutionIsSuspended() + throws Exception { + ScheduledExecutorService schedulerPool = Executors.newSingleThreadScheduledExecutor(); + ExecutorService workerPool = Executors.newFixedThreadPool(2); + TriggerScheduler triggerScheduler = new TriggerScheduler( + new InMemoryTriggerStore(), schedulerPool, workerPool, 10L); + ChainDefinition definition = createConfirmDefinition(); + ChainExecutor executor = new ChainExecutor( + ignored -> definition, + new InMemoryChainStateRepository(), + new InMemoryNodeStateRepository(), + triggerScheduler); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future> result = caller.submit( + () -> executor.executeWithoutSuspension( + definition.getId(), Collections.emptyMap())); + try { + result.get(3, TimeUnit.SECONDS); + Assert.fail("suspended workflow must fail"); + } catch (ExecutionException exception) { + Assert.assertTrue( + String.valueOf(exception.getCause().getMessage()) + .contains("Execution failed")); + } + } finally { + caller.shutdownNow(); + triggerScheduler.shutdown(); + } + } + /** * 验证实例初始化会在入口触发器创建前持久化工作流定义 ID。 * @@ -512,6 +551,36 @@ public class ChainExecutorConcurrencyTest { return definition; } + /** + * 创建包含内部确认节点的测试 Workflow。 + * + * @return 会进入挂起状态的 Workflow 定义 + */ + private ChainDefinition createConfirmDefinition() { + ChainDefinition definition = new ChainDefinition(); + definition.setId("non-suspending-confirm-test"); + StartNode start = new StartNode(); + start.setId("start"); + ConfirmNode confirm = new ConfirmNode(); + confirm.setId("confirm"); + EndNode end = new EndNode(); + end.setId("end"); + Edge first = new Edge(); + first.setId("start-to-confirm"); + first.setSource("start"); + first.setTarget("confirm"); + Edge second = new Edge(); + second.setId("confirm-to-end"); + second.setSource("confirm"); + second.setTarget("end"); + definition.addNode(start); + definition.addNode(confirm); + definition.addNode(end); + definition.addEdge(first); + definition.addEdge(second); + return definition; + } + /** * 创建用于取消传播验证的工作流。 * From 9612c5bd62deb90099e0bdfcd64a9715b974ce7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 19 Aug 2026 21:51:27 +0800 Subject: [PATCH 30/33] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20Agent=20?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E5=B7=A5=E4=BD=9C=E5=8C=BA=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 提供受控文件读写、补丁、Shell 与归档能力 - 补齐路径、配额、命令审批和进程清理边界 --- easy-agents-agent-runtime/pom.xml | 5 + .../tool/operate/AgentOperateToolAdapter.java | 74 +- .../tool/operate/AgentOperateToolSpec.java | 201 ++- .../tool/operate/AgentOperateToolType.java | 5 + .../runtime/tool/operate/ApplyPatchTool.java | 300 ++++ .../tool/operate/ControlledShellTool.java | 777 ++++++++++ .../operate/SafeArchiveCommandExecutor.java | 1319 +++++++++++++++++ .../tool/operate/SafeReadFileTool.java | 290 ++++ .../tool/operate/SafeWriteFileTool.java | 325 ++++ .../operate/ShellCommandOptionValidator.java | 601 ++++++++ .../operate/ShellProcessGroupSupport.java | 150 ++ .../tool/operate/UnifiedPatchParser.java | 287 ++++ .../tool/operate/WorkspacePathGuard.java | 311 ++++ .../tool/operate/WorkspaceQuotaGuard.java | 263 ++++ .../tool/operate/WorkspaceQuotaHook.java | 74 + .../tool/operate/WorkspaceQuotaLimits.java | 78 + .../tool/operate/WorkspaceTextFiles.java | 269 ++++ .../tool/operate/WorkspaceToolException.java | 57 + .../tool/operate/WorkspaceToolResults.java | 58 + .../AgentScopeStatefulRuntimeTest.java | 348 ++++- .../operate/AgentOperateToolAdapterTest.java | 40 +- .../tool/operate/ApplyPatchToolTest.java | 142 ++ .../tool/operate/ControlledShellToolTest.java | 347 +++++ .../SafeArchiveCommandExecutorTest.java | 228 +++ .../operate/ShellProcessGroupSupportTest.java | 35 + .../tool/operate/WorkspaceFileToolsTest.java | 188 +++ .../tool/operate/WorkspacePathGuardTest.java | 121 ++ 27 files changed, 6842 insertions(+), 51 deletions(-) create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchTool.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ControlledShellTool.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutor.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeReadFileTool.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeWriteFileTool.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellCommandOptionValidator.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupport.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/UnifiedPatchParser.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuard.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaGuard.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaHook.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaLimits.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceTextFiles.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolException.java create mode 100644 easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolResults.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchToolTest.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ControlledShellToolTest.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutorTest.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupportTest.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspaceFileToolsTest.java create mode 100644 easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuardTest.java diff --git a/easy-agents-agent-runtime/pom.xml b/easy-agents-agent-runtime/pom.xml index 234d859..e2112bd 100644 --- a/easy-agents-agent-runtime/pom.xml +++ b/easy-agents-agent-runtime/pom.xml @@ -28,6 +28,11 @@ fastjson2 + + org.apache.commons + commons-compress + + com.anthropic anthropic-java diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java index e7a64c1..dc33402 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java @@ -6,21 +6,15 @@ import com.easyagents.agent.runtime.tool.AgentToolCategory; import com.easyagents.agent.runtime.tool.AgentToolSpec; import com.easyagents.agent.runtime.tool.AgentToolVisibility; import io.agentscope.core.tool.Toolkit; -import io.agentscope.core.tool.coding.ShellCommandTool; -import io.agentscope.core.tool.file.ReadFileTool; -import io.agentscope.core.tool.file.WriteFileTool; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.*; /** * AgentScope 内置操作工具适配器。 * - *

该适配器只负责将 Easy-Agents 的操作工具声明转换为 AgentScope Toolkit 中的原生工具。 - * Shell 工具的人工审批不使用 AgentScope {@code ShellCommandTool} 的同步 callback,而是通过 - * Easy-Agents 现有 {@code ToolHitlInterceptor} 统一处理,以保持 SSE 暂停、恢复和审计语义一致。 + *

该适配器将 Easy-Agents 的操作工具声明转换为与 AgentScope 1.x 工具名和 Schema 兼容的 + * 受控实现。Shell 人工审批继续通过 Easy-Agents {@code ToolHitlInterceptor} 处理,以保持 + * SSE 暂停、恢复和审计语义一致。 */ public class AgentOperateToolAdapter { @@ -28,6 +22,7 @@ public class AgentOperateToolAdapter { public static final String LIST_DIRECTORY_TOOL = "list_directory"; public static final String WRITE_TEXT_FILE_TOOL = "write_text_file"; public static final String INSERT_TEXT_FILE_TOOL = "insert_text_file"; + public static final String APPLY_PATCH_TOOL = "apply_patch"; public static final String EXECUTE_SHELL_COMMAND_TOOL = "execute_shell_command"; /** @@ -75,6 +70,7 @@ public class AgentOperateToolAdapter { names.add(WRITE_TEXT_FILE_TOOL); names.add(INSERT_TEXT_FILE_TOOL); } + case PATCH -> names.add(APPLY_PATCH_TOOL); case SHELL -> names.add(EXECUTE_SHELL_COMMAND_TOOL); default -> { } @@ -88,54 +84,63 @@ public class AgentOperateToolAdapter { if (type == null) { throw new AgentRuntimeException("Agent operate tool type is required."); } - Path baseDir = validateBaseDir(spec); + WorkspacePathGuard pathGuard = createPathGuard(spec); + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard( + pathGuard, spec.getWorkspaceQuotaLimits(), spec.getWorkspaceQuotaHook()); switch (type) { case READ_FILE -> { assertNoToolConflict(toolkit, VIEW_TEXT_FILE_TOOL); assertNoToolConflict(toolkit, LIST_DIRECTORY_TOOL); - toolkit.registerTool(new ReadFileTool(baseDir.toString())); + SafeReadFileTool readFileTool = new SafeReadFileTool(pathGuard, quotaGuard); + toolkit.registerAgentTool(readFileTool.viewTextFileTool()); + toolkit.registerAgentTool(readFileTool.listDirectoryTool()); toolSpecs.add(toolSpec(spec, VIEW_TEXT_FILE_TOOL, "View text file content.", false)); toolSpecs.add(toolSpec(spec, LIST_DIRECTORY_TOOL, "List files and directories.", false)); } case WRITE_FILE -> { assertNoToolConflict(toolkit, WRITE_TEXT_FILE_TOOL); assertNoToolConflict(toolkit, INSERT_TEXT_FILE_TOOL); - toolkit.registerTool(new WriteFileTool(baseDir.toString())); - toolSpecs.add(toolSpec(spec, WRITE_TEXT_FILE_TOOL, "Write or replace text file content.", true)); - toolSpecs.add(toolSpec(spec, INSERT_TEXT_FILE_TOOL, "Insert text into a file.", true)); + SafeWriteFileTool writeFileTool = new SafeWriteFileTool(pathGuard, quotaGuard); + toolkit.registerAgentTool(writeFileTool.writeTextFileTool()); + toolkit.registerAgentTool(writeFileTool.insertTextFileTool()); + toolSpecs.add(toolSpec(spec, WRITE_TEXT_FILE_TOOL, "Write or replace text file content.", false)); + toolSpecs.add(toolSpec(spec, INSERT_TEXT_FILE_TOOL, "Insert text into a file.", false)); + } + case PATCH -> { + assertNoToolConflict(toolkit, APPLY_PATCH_TOOL); + toolkit.registerAgentTool(new ApplyPatchTool( + pathGuard, quotaGuard, spec.getPatchMaxSize(), + spec.getPatchMaxFiles(), spec.getPatchMaxAffectedBytes())); + toolSpecs.add(toolSpec(spec, APPLY_PATCH_TOOL, "Apply a workspace text patch.", false)); } case SHELL -> { assertNoToolConflict(toolkit, EXECUTE_SHELL_COMMAND_TOOL); - Charset charset = parseCharset(spec); - toolkit.registerAgentTool(new ShellCommandTool(baseDir.toString(), spec.getShellAllowedCommands(), null, - null, charset)); - toolSpecs.add(toolSpec(spec, EXECUTE_SHELL_COMMAND_TOOL, "Execute shell command.", true)); + ControlledShellTool shellTool = new ControlledShellTool(pathGuard, quotaGuard, spec); + toolkit.registerAgentTool(shellTool); + AgentToolSpec shellToolSpec = toolSpec( + spec, EXECUTE_SHELL_COMMAND_TOOL, "Execute shell command.", true); + shellToolSpec.setApprovalPolicy(shellTool::approvalEvaluation); + toolSpecs.add(shellToolSpec); } default -> throw new AgentRuntimeException("Unsupported agent operate tool type: " + type); } } - private Path validateBaseDir(AgentOperateToolSpec spec) { + private WorkspacePathGuard createPathGuard(AgentOperateToolSpec spec) { String baseDir = spec.getBaseDir(); if (baseDir == null || baseDir.isBlank()) { throw new AgentRuntimeException("Agent operate tool baseDir is required."); } - Path path = Path.of(baseDir).toAbsolutePath().normalize(); if (!Path.of(baseDir).isAbsolute()) { - throw new AgentRuntimeException("Agent operate tool baseDir must be an absolute path: " + baseDir); - } - return path; - } - - private Charset parseCharset(AgentOperateToolSpec spec) { - String charsetName = spec.getShellCharset(); - if (charsetName == null || charsetName.isBlank()) { - return StandardCharsets.UTF_8; + throw new AgentRuntimeException("Agent operate tool baseDir must be an absolute path."); } try { - return Charset.forName(charsetName.trim()); - } catch (Exception error) { - throw new AgentRuntimeException("Invalid shell charset: " + charsetName, error); + return new WorkspacePathGuard(Path.of(baseDir).toAbsolutePath().normalize()); + } catch (RuntimeException error) { + if (error instanceof AgentRuntimeException runtimeError) { + throw runtimeError; + } + throw new AgentRuntimeException("Agent operate tool baseDir is invalid.", error); } } @@ -172,7 +177,10 @@ public class AgentOperateToolAdapter { Map metadata = new LinkedHashMap<>(); metadata.put("operateTool", true); metadata.put("operateToolType", spec.getType().name()); - metadata.put("baseDir", spec.getBaseDir()); + if (spec.getType() == AgentOperateToolType.SHELL) { + metadata.put("forceApprovalCommands", List.of("rm")); + metadata.put("forceApprovalCommandArgument", "command"); + } return metadata; } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolSpec.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolSpec.java index 78df6de..e3cc53d 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolSpec.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolSpec.java @@ -4,13 +4,13 @@ import com.easyagents.agent.runtime.hitl.AgentToolApprovalRequest; import java.util.LinkedHashSet; import java.util.Set; +import java.time.Duration; /** * Agent 操作类工具声明。 * - *

操作类工具是 runtime 直接适配的 AgentScope 内置工具,用于读文件、写文件和执行 Shell。 - * 这些工具直接作用于后端 JVM 所在宿主环境,调用方必须按 agent、session 或 user 维度传入受控 - * 的绝对工作目录。 + *

操作类工具由 runtime 适配为与 AgentScope 1.x 契约兼容的受控工具。调用方必须按 + * agent、session 或 user 维度传入独立的绝对工作目录,并通过配额与 Shell 参数限制资源使用。 */ public class AgentOperateToolSpec { @@ -19,8 +19,18 @@ public class AgentOperateToolSpec { private String baseDir; private Boolean approvalRequired; private AgentToolApprovalRequest approvalRequest; - private Set shellAllowedCommands = new LinkedHashSet<>(); - private String shellCharset; + private WorkspaceQuotaLimits workspaceQuotaLimits = WorkspaceQuotaLimits.unlimited(); + private transient WorkspaceQuotaHook workspaceQuotaHook = WorkspaceQuotaHook.noop(); + private Set shellAllowedCommands = new LinkedHashSet<>(ControlledShellTool.DEFAULT_ALLOWED_COMMANDS); + private String shellCharset = "UTF-8"; + private Duration shellDefaultTimeout = Duration.ofSeconds(60); + private Duration shellMaxTimeout = Duration.ofSeconds(300); + private int shellMaxCommandLength = 4096; + private long shellMaxOutputSize = 1024L * 1024L; + private int shellMaxConcurrency = 2; + private long patchMaxSize = 1024L * 1024L; + private int patchMaxFiles = 100; + private long patchMaxAffectedBytes = 16L * 1024L * 1024L; /** * 获取操作工具类型。 @@ -76,6 +86,43 @@ public class AgentOperateToolSpec { this.baseDir = baseDir; } + /** + * 获取工作区配额。 + * + * @return 工作区配额 + */ + public WorkspaceQuotaLimits getWorkspaceQuotaLimits() { + return workspaceQuotaLimits; + } + + /** + * 设置工作区配额。 + * + * @param workspaceQuotaLimits 工作区配额,null 表示不限制 + */ + public void setWorkspaceQuotaLimits(WorkspaceQuotaLimits workspaceQuotaLimits) { + this.workspaceQuotaLimits = workspaceQuotaLimits == null + ? WorkspaceQuotaLimits.unlimited() : workspaceQuotaLimits; + } + + /** + * 获取业务侧附加配额校验 Hook。 + * + * @return 配额校验 Hook + */ + public WorkspaceQuotaHook getWorkspaceQuotaHook() { + return workspaceQuotaHook; + } + + /** + * 设置业务侧附加配额校验 Hook。 + * + * @param workspaceQuotaHook 配额校验 Hook,null 表示无附加校验 + */ + public void setWorkspaceQuotaHook(WorkspaceQuotaHook workspaceQuotaHook) { + this.workspaceQuotaHook = workspaceQuotaHook == null ? WorkspaceQuotaHook.noop() : workspaceQuotaHook; + } + /** * 获取审批开关覆盖值。 * @@ -147,4 +194,148 @@ public class AgentOperateToolSpec { public void setShellCharset(String shellCharset) { this.shellCharset = shellCharset; } + + /** + * 获取 Shell 默认超时。 + * + * @return 默认超时 + */ + public Duration getShellDefaultTimeout() { + return shellDefaultTimeout; + } + + /** + * 设置 Shell 默认超时。 + * + * @param shellDefaultTimeout 默认超时 + */ + public void setShellDefaultTimeout(Duration shellDefaultTimeout) { + this.shellDefaultTimeout = shellDefaultTimeout; + } + + /** + * 获取 Shell 最大超时。 + * + * @return 最大超时 + */ + public Duration getShellMaxTimeout() { + return shellMaxTimeout; + } + + /** + * 设置 Shell 最大超时。 + * + * @param shellMaxTimeout 最大超时 + */ + public void setShellMaxTimeout(Duration shellMaxTimeout) { + this.shellMaxTimeout = shellMaxTimeout; + } + + /** + * 获取 Shell 命令最大长度。 + * + * @return 最大字符数 + */ + public int getShellMaxCommandLength() { + return shellMaxCommandLength; + } + + /** + * 设置 Shell 命令最大长度。 + * + * @param shellMaxCommandLength 最大字符数 + */ + public void setShellMaxCommandLength(int shellMaxCommandLength) { + this.shellMaxCommandLength = shellMaxCommandLength; + } + + /** + * 获取 Shell 单次标准输出和错误输出各自的最大字节数。 + * + * @return 最大字节数 + */ + public long getShellMaxOutputSize() { + return shellMaxOutputSize; + } + + /** + * 设置 Shell 单次标准输出和错误输出各自的最大字节数。 + * + * @param shellMaxOutputSize 最大字节数 + */ + public void setShellMaxOutputSize(long shellMaxOutputSize) { + this.shellMaxOutputSize = shellMaxOutputSize; + } + + /** + * 获取 JVM 实例级 Shell 最大并发数。 + * + * @return 最大并发数 + */ + public int getShellMaxConcurrency() { + return shellMaxConcurrency; + } + + /** + * 设置 JVM 实例级 Shell 最大并发数。 + * + * @param shellMaxConcurrency 最大并发数 + */ + public void setShellMaxConcurrency(int shellMaxConcurrency) { + this.shellMaxConcurrency = shellMaxConcurrency; + } + + /** + * 获取 Patch 输入最大字节数。 + * + * @return 最大字节数 + */ + public long getPatchMaxSize() { + return patchMaxSize; + } + + /** + * 设置 Patch 输入最大字节数。 + * + * @param patchMaxSize 最大字节数 + */ + public void setPatchMaxSize(long patchMaxSize) { + this.patchMaxSize = patchMaxSize; + } + + /** + * 获取 Patch 最大影响文件数。 + * + * @return 最大文件数 + */ + public int getPatchMaxFiles() { + return patchMaxFiles; + } + + /** + * 设置 Patch 最大影响文件数。 + * + * @param patchMaxFiles 最大文件数 + */ + public void setPatchMaxFiles(int patchMaxFiles) { + this.patchMaxFiles = patchMaxFiles; + } + + /** + * 获取 Patch 影响内容最大总字节数。 + * + * @return 最大字节数 + */ + public long getPatchMaxAffectedBytes() { + return patchMaxAffectedBytes; + } + + /** + * 设置 Patch 影响内容最大总字节数。 + * + * @param patchMaxAffectedBytes 最大字节数 + */ + public void setPatchMaxAffectedBytes(long patchMaxAffectedBytes) { + this.patchMaxAffectedBytes = patchMaxAffectedBytes; + } } diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolType.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolType.java index e566420..b7ea95b 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolType.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolType.java @@ -15,6 +15,11 @@ public enum AgentOperateToolType { */ WRITE_FILE, + /** + * 以补丁方式新增、更新或删除工作区文本文件。 + */ + PATCH, + /** * 在服务进程所在宿主环境执行 Shell 命令。 */ diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchTool.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchTool.java new file mode 100644 index 0000000..67c4667 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchTool.java @@ -0,0 +1,300 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.AgentTool; +import io.agentscope.core.tool.ToolCallParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 有界 unified diff / context hunk 工作区补丁工具。 + */ +final class ApplyPatchTool implements AgentTool { + + private static final Logger logger = LoggerFactory.getLogger(ApplyPatchTool.class); + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + private final long maxPatchSize; + private final int maxFiles; + private final long maxAffectedBytes; + + /** + * 创建补丁工具。 + * + * @param pathGuard 路径保护器 + * @param quotaGuard 配额保护器 + * @param maxPatchSize Patch 输入最大字节数 + * @param maxFiles 单次最大影响文件数 + * @param maxAffectedBytes 原内容与新内容合计最大字节数 + */ + ApplyPatchTool(WorkspacePathGuard pathGuard, + WorkspaceQuotaGuard quotaGuard, + long maxPatchSize, + int maxFiles, + long maxAffectedBytes) { + if (maxPatchSize <= 0 || maxFiles <= 0 || maxAffectedBytes <= 0) { + throw new WorkspaceToolException("WORKSPACE_CONFIG_INVALID", "Patch limits must be positive.", false); + } + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + this.maxPatchSize = maxPatchSize; + this.maxFiles = maxFiles; + this.maxAffectedBytes = maxAffectedBytes; + } + + /** + * 获取工具名。 + * + * @return `apply_patch` + */ + @Override + public String getName() { + return AgentOperateToolAdapter.APPLY_PATCH_TOOL; + } + + /** + * 获取工具描述。 + * + * @return 工具描述 + */ + @Override + public String getDescription() { + return "Apply a bounded unified diff to workspace-relative UTF-8 text files atomically per file."; + } + + /** + * 获取参数 Schema。 + * + * @return JSON Schema + */ + @Override + public Map getParameters() { + return Map.of( + "type", "object", + "properties", Map.of("patch", Map.of( + "type", "string", + "description", "Unified diff or *** Begin Patch context patch")), + "required", List.of("patch")); + } + + /** + * 解析、预检并应用补丁。 + * + * @param param Tool 调用参数 + * @return Tool 结果 + */ + @Override + public Mono callAsync(ToolCallParam param) { + return Mono.fromCallable(() -> apply(param)).subscribeOn(Schedulers.boundedElastic()); + } + + private ToolResultBlock apply(ToolCallParam param) { + try { + Object value = param == null ? null : param.getInput().get("patch"); + if (!(value instanceof String patch) || patch.isBlank()) { + throw new WorkspaceToolException("PATCH_INVALID", "Missing required string parameter: patch.", false); + } + if (patch.getBytes(StandardCharsets.UTF_8).length > maxPatchSize) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Patch input exceeds the configured maximum size.", false); + } + List patches = UnifiedPatchParser.parse(patch); + if (patches.isEmpty()) { + throw new WorkspaceToolException("PATCH_INVALID", "Patch does not contain file changes.", false); + } + if (patches.size() > maxFiles) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Patch affects too many files.", false); + } + PatchPlan plan = prepare(patches); + commit(plan); + return ToolResultBlock.text("Patch applied successfully: " + plan.changes().size() + + " file(s), " + plan.addedLines() + " insertion(s), " + + plan.deletedLines() + " deletion(s)."); + } catch (AgentRuntimeException error) { + return WorkspaceToolResults.error(error); + } catch (RuntimeException error) { + return WorkspaceToolResults.error( + new AgentRuntimeException("Unexpected patch execution failure.", error)); + } + } + + private PatchPlan prepare(List patches) { + Map originals = new LinkedHashMap<>(); + Map desired = new LinkedHashMap<>(); + Map resultingSizes = new LinkedHashMap<>(); + long affectedBytes = 0; + int addedLines = 0; + int deletedLines = 0; + for (FilePatch patch : patches) { + Path target = patch.type() == PatchType.ADD + ? pathGuard.resolveForWrite(patch.path()) : pathGuard.resolveExistingFile(patch.path()); + if (originals.containsKey(target)) { + throw new WorkspaceToolException("PATCH_INVALID", "Patch contains a duplicate target.", false); + } + byte[] original = null; + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + quotaGuard.validateFullRead(target); + original = readBytes(target); + } + if (patch.type() == PatchType.ADD && original != null) { + throw new WorkspaceToolException("PATCH_CONFLICT", "Patch add target already exists.", false); + } + String current = original == null ? "" : WorkspaceTextFiles.decodeUtf8(original); + String updated = UnifiedPatchParser.apply(patch, current); + byte[] next = null; + if (patch.type() != PatchType.DELETE) { + next = updated.getBytes(StandardCharsets.UTF_8); + } + affectedBytes = addBounded(affectedBytes, original == null ? 0 : original.length); + affectedBytes = addBounded(affectedBytes, next == null ? 0 : next.length); + originals.put(target, original); + desired.put(target, next); + resultingSizes.put(target, next == null ? -1L : (long) next.length); + addedLines += patch.addedLines(); + deletedLines += patch.deletedLines(); + } + quotaGuard.validateBatch(resultingSizes); + return new PatchPlan(originals, desired, List.copyOf(desired.keySet()), addedLines, deletedLines); + } + + private long addBounded(long left, long right) { + long value; + try { + value = Math.addExact(left, right); + } catch (ArithmeticException error) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Patch affected content exceeds the configured maximum size.", false, error); + } + if (value > maxAffectedBytes) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Patch affected content exceeds the configured maximum size.", false); + } + return value; + } + + private void commit(PatchPlan plan) { + List committed = new ArrayList<>(); + try { + for (Path target : plan.changes()) { + byte[] next = plan.desired().get(target); + pathGuard.revalidate(target); + if (next == null) { + Files.delete(target); + } else { + WorkspaceTextFiles.atomicWrite(pathGuard, target, next); + } + committed.add(target); + } + } catch (Exception commitError) { + Collections.reverse(committed); + Exception rollbackError = null; + for (Path target : committed) { + try { + byte[] original = plan.originals().get(target); + if (original == null) { + Files.deleteIfExists(target); + } else { + WorkspaceTextFiles.atomicWrite(pathGuard, target, original); + } + } catch (Exception error) { + if (rollbackError == null) { + rollbackError = error; + } else { + rollbackError.addSuppressed(error); + } + } + } + if (rollbackError != null) { + commitError.addSuppressed(rollbackError); + logger.error("Patch commit and rollback failed; workspace requires inspection", commitError); + throw new WorkspaceToolException("PATCH_ROLLBACK_FAILED", + "Patch commit and rollback failed; workspace requires inspection.", false, commitError); + } + logger.error("Patch commit failed and was rolled back", commitError); + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Patch commit failed and all changes were rolled back.", true, commitError); + } + } + + private byte[] readBytes(Path target) { + return WorkspaceTextFiles.readUtf8(target).getBytes(StandardCharsets.UTF_8); + } + + /** + * 补丁事务计划。 + * + * @param originals 提交前原内容 + * @param desired 提交后内容,null 表示删除 + * @param changes 有序目标列表 + * @param addedLines 新增行数 + * @param deletedLines 删除行数 + */ + private record PatchPlan(Map originals, + Map desired, + List changes, + int addedLines, + int deletedLines) { + } + + /** + * 文件变更类型。 + */ + enum PatchType { + /** 新增文件。 */ + ADD, + /** 更新文件。 */ + UPDATE, + /** 删除文件。 */ + DELETE + } + + /** + * 单文件补丁。 + * + * @param type 变更类型 + * @param path 工作区相对路径 + * @param hunks 上下文块 + * @param addedLines 新增行数 + * @param deletedLines 删除行数 + */ + record FilePatch(PatchType type, + String path, + List hunks, + int addedLines, + int deletedLines) { + } + + /** + * 单个上下文块。 + * + * @param oldStart unified diff 声明的原起始行,可空 + * @param lines 上下文行 + */ + record Hunk(Integer oldStart, List lines) { + } + + /** + * 上下文行。 + * + * @param kind 空格表示上下文,减号表示删除,加号表示新增 + * @param text 行内容 + */ + record DiffLine(char kind, String text) { + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ControlledShellTool.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ControlledShellTool.java new file mode 100644 index 0000000..a0e3ad2 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ControlledShellTool.java @@ -0,0 +1,777 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import com.easyagents.agent.runtime.hitl.AgentToolApprovalEvaluation; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.AgentTool; +import io.agentscope.core.tool.ToolCallParam; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +/** + * 不经过系统 Shell 解释器的受控命令执行工具。 + * + *

命令先按受限引号规则拆分为参数,再直接交给 {@link ProcessBuilder}。因此管道、重定向、 + * 命令替换和环境变量展开既会被显式拒绝,也不会被二次解释。 + */ +public final class ControlledShellTool implements AgentTool { + + /** L22 首版固定命令白名单。 */ + public static final Set DEFAULT_ALLOWED_COMMANDS = Set.of( + "pwd", "ls", "cat", "head", "tail", "wc", "grep", "rg", "sed", "awk", "sort", "uniq", + "cut", "tr", "basename", "dirname", "stat", "file", "date", "sha256sum", "shasum", "jq", + "diff", "cmp", "du", "tree", + "mkdir", "touch", "cp", "mv", "rm", "python", "python3", "node", + "gzip", "gunzip", "zip", "unzip", "tar", + "pandoc", "soffice", "pdftoppm", "pdfinfo", "pdftotext", "pdfimages", "qpdf"); + + private static final Set APPROVAL_REQUIRED_COMMANDS = Set.of( + "mkdir", "touch", "cp", "mv", "gzip", "gunzip", "zip", "unzip", "tar", + "pandoc", "soffice", "pdftoppm", "pdfimages", "qpdf"); + + private static final Map INSTANCE_LIMITERS = new ConcurrentHashMap<>(); + private static final Map OUTPUT_EXECUTORS = new ConcurrentHashMap<>(); + private static final Map ACTIVE_PROCESS_TREES = new ConcurrentHashMap<>(); + private static final AtomicInteger OUTPUT_THREAD_SEQUENCE = new AtomicInteger(); + private static final String FORBIDDEN_METACHARACTERS = ";|&><`$"; + private static final String TRUSTED_EXECUTABLE_PATH = "/usr/local/bin:/usr/bin:/bin"; + + static { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + for (Map.Entry entry : ACTIVE_PROCESS_TREES.entrySet()) { + ActiveProcess active = entry.getValue(); + active.processGroupSupport().terminate(active.processGroupId()); + terminateProcessTreeNow(entry.getKey(), active.observedDescendants()); + } + OUTPUT_EXECUTORS.values().forEach(ExecutorService::shutdownNow); + }, "easyagents-shell-shutdown")); + } + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + private final Set allowedCommands; + private final int defaultTimeoutSeconds; + private final int maxTimeoutSeconds; + private final int maxCommandLength; + private final int maxOutputSize; + private final Semaphore limiter; + private final ExecutorService outputExecutor; + private final ShellCommandOptionValidator optionValidator; + private final SafeArchiveCommandExecutor archiveCommandExecutor; + private final ShellProcessGroupSupport processGroupSupport; + + /** + * 创建受控 Shell 工具。 + * + * @param pathGuard 路径保护器 + * @param quotaGuard 配额保护器 + * @param spec 操作工具配置 + */ + public ControlledShellTool(WorkspacePathGuard pathGuard, + WorkspaceQuotaGuard quotaGuard, + AgentOperateToolSpec spec) { + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + this.allowedCommands = validateAllowedCommands(spec.getShellAllowedCommands()); + this.defaultTimeoutSeconds = seconds(spec.getShellDefaultTimeout(), "shellDefaultTimeout"); + this.maxTimeoutSeconds = seconds(spec.getShellMaxTimeout(), "shellMaxTimeout"); + if (defaultTimeoutSeconds > maxTimeoutSeconds) { + throw new AgentRuntimeException("Shell default timeout must not exceed max timeout."); + } + if (spec.getShellMaxCommandLength() <= 0 || spec.getShellMaxOutputSize() <= 0 + || spec.getShellMaxOutputSize() > Integer.MAX_VALUE || spec.getShellMaxConcurrency() <= 0 + || spec.getShellMaxConcurrency() > 64) { + throw new AgentRuntimeException("Shell limits must be positive and output size must fit in memory."); + } + if (spec.getShellCharset() != null && !spec.getShellCharset().isBlank() + && !"UTF-8".equalsIgnoreCase(spec.getShellCharset().trim())) { + throw new AgentRuntimeException("Shell charset must be UTF-8."); + } + this.maxCommandLength = spec.getShellMaxCommandLength(); + this.maxOutputSize = (int) spec.getShellMaxOutputSize(); + this.limiter = INSTANCE_LIMITERS.computeIfAbsent(spec.getShellMaxConcurrency(), Semaphore::new); + this.outputExecutor = OUTPUT_EXECUTORS.computeIfAbsent( + spec.getShellMaxConcurrency(), ControlledShellTool::createOutputExecutor); + this.optionValidator = new ShellCommandOptionValidator(pathGuard); + this.archiveCommandExecutor = new SafeArchiveCommandExecutor(pathGuard, quotaGuard, maxOutputSize); + this.processGroupSupport = ShellProcessGroupSupport.detect(); + } + + /** + * 获取工具名。 + * + * @return `execute_shell_command` + */ + @Override + public String getName() { + return AgentOperateToolAdapter.EXECUTE_SHELL_COMMAND_TOOL; + } + + /** + * 获取工具描述。 + * + * @return 工具描述 + */ + @Override + public String getDescription() { + return "Execute one allowlisted command in the workspace without shell operators or host path access."; + } + + /** + * 获取与 AgentScope 1.x 兼容的参数 Schema。 + * + * @return JSON Schema + */ + @Override + public Map getParameters() { + return Map.of( + "type", "object", + "properties", Map.of( + "command", Map.of("type", "string", "description", "The single command to execute"), + "timeout", Map.of("type", "integer", "description", "Execution timeout in seconds"), + "charset", Map.of("type", "string", "description", "Must be UTF-8 when supplied")), + "required", List.of("command")); + } + + /** + * 校验并异步执行命令。 + * + * @param param Tool 调用参数 + * @return Tool 结果 + */ + @Override + public Mono callAsync(ToolCallParam param) { + return Mono.fromCallable(() -> execute(param)).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * 在 HITL 事件生成前校验命令并计算单次调用的审批策略。 + * + *

无效命令不弹出审批,随后由工具调用返回结构化拒绝结果。Python/Node 脚本以 + * 脚本内容和参数的摘要作为本轮复用作用域,脚本变化后必须重新审批。

+ * + * @param toolInput Shell 工具入参 + * @return 动态审批判定 + */ + public AgentToolApprovalEvaluation approvalEvaluation(Map toolInput) { + try { + String command = requiredCommand(toolInput); + List arguments = parse(command); + validate(arguments); + String executable = arguments.get(0); + if ("rm".equals(executable)) { + return AgentToolApprovalEvaluation.valid(true, true, null); + } + if (Set.of("python", "python3", "node").contains(executable)) { + return AgentToolApprovalEvaluation.valid(true, false, scriptApprovalScope(arguments)); + } + if ("pdftotext".equals(executable)) { + boolean stdoutOnly = arguments.size() >= 3 && "-".equals(arguments.get(arguments.size() - 1)); + return AgentToolApprovalEvaluation.valid(!stdoutOnly, false, null); + } + return AgentToolApprovalEvaluation.valid( + APPROVAL_REQUIRED_COMMANDS.contains(executable), false, null); + } catch (RuntimeException error) { + return AgentToolApprovalEvaluation.invalid(); + } + } + + private ToolResultBlock execute(ToolCallParam param) { + boolean acquired = false; + Process process = null; + long processGroupId = -1; + Set observedDescendants = ConcurrentHashMap.newKeySet(); + try { + String command = requiredCommand(param); + int timeout = requestedTimeout(param); + validateCharset(param); + List arguments = parse(command); + validate(arguments); + quotaGuard.validateCurrentUsage(); + acquired = limiter.tryAcquire(Math.min(timeout, defaultTimeoutSeconds), TimeUnit.SECONDS); + if (!acquired) { + return WorkspaceToolResults.error( + "SHELL_CONCURRENCY_LIMIT", "Shell execution queue is full.", true); + } + long startedAt = System.nanoTime(); + if (SafeArchiveCommandExecutor.COMMANDS.contains(arguments.get(0))) { + SafeArchiveCommandExecutor.ArchiveExecutionResult archiveResult = + archiveCommandExecutor.execute(arguments, + startedAt + TimeUnit.SECONDS.toNanos(timeout)); + quotaGuard.validateCurrentUsage(); + long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + return result(0, + new BoundedOutput(archiveResult.output(), archiveResult.truncated()), + new BoundedOutput("", false), null, null, false, durationMillis); + } + ProcessBuilder processBuilder = new ProcessBuilder(processGroupSupport.wrap(arguments)); + processBuilder.directory(pathGuard.root().toFile()); + sanitizeEnvironment(processBuilder.environment()); + process = processBuilder.start(); + processGroupId = processGroupSupport.enabled() ? process.pid() : -1; + ACTIVE_PROCESS_TREES.put(process, + new ActiveProcess(observedDescendants, processGroupSupport, processGroupId)); + CompletableFuture stdout = readBounded(process.getInputStream()); + CompletableFuture stderr = readBounded(process.getErrorStream()); + boolean completed; + try { + completed = waitForProcess(process, timeout, observedDescendants); + } catch (InterruptedException interrupted) { + terminateProcessTree(process, observedDescendants, processGroupId); + Thread.currentThread().interrupt(); + return WorkspaceToolResults.error("SHELL_INTERRUPTED", "Shell command was interrupted.", true); + } + if (!completed) { + terminateProcessTree(process, observedDescendants, processGroupId); + } else { + // 白名单脚本不允许在 Tool 正常返回后遗留后台子进程。 + processGroupSupport.terminate(processGroupId); + terminateObservedDescendants(observedDescendants); + } + BoundedOutput stdoutValue = awaitOutput(stdout); + BoundedOutput stderrValue = awaitOutput(stderr); + quotaGuard.validateCurrentUsage(); + long durationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + if (!completed) { + return result(-1, stdoutValue, stderrValue, + "SHELL_TIMEOUT", "Shell command exceeded " + timeout + " seconds.", true, durationMillis); + } + return result(process.exitValue(), stdoutValue, stderrValue, null, null, false, durationMillis); + } catch (WorkspaceToolException error) { + return WorkspaceToolResults.error(error); + } catch (AgentRuntimeException error) { + return WorkspaceToolResults.error("SHELL_COMMAND_DENIED", error.getMessage(), false); + } catch (IOException error) { + return WorkspaceToolResults.error( + "SHELL_EXECUTION_FAILED", "Command is unavailable or could not be started.", false); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + return WorkspaceToolResults.error("SHELL_INTERRUPTED", "Shell execution queue wait was interrupted.", true); + } catch (RuntimeException error) { + return WorkspaceToolResults.error( + new AgentRuntimeException("Unexpected shell execution failure.", error)); + } finally { + if (process != null) { + if (process.isAlive()) { + terminateProcessTree(process, observedDescendants, processGroupId); + } else { + processGroupSupport.terminate(processGroupId); + terminateObservedDescendants(observedDescendants); + } + ACTIVE_PROCESS_TREES.remove(process); + } + if (acquired) { + limiter.release(); + } + } + } + + private List parse(String command) { + List tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + char quote = 0; + boolean escaping = false; + for (int index = 0; index < command.length(); index++) { + char character = command.charAt(index); + if (character == '\n' || character == '\r' || character == '\0' + || Character.isISOControl(character)) { + throw new AgentRuntimeException("Shell control characters are not allowed."); + } + if (FORBIDDEN_METACHARACTERS.indexOf(character) >= 0 || character == '~') { + throw new AgentRuntimeException("Shell operators, substitutions, and expansions are not allowed."); + } + if (escaping) { + current.append(character); + escaping = false; + } else if (character == '\\' && quote != '\'') { + escaping = true; + } else if ((character == '\'' || character == '"')) { + if (quote == 0) { + quote = character; + } else if (quote == character) { + quote = 0; + } else { + current.append(character); + } + } else if (Character.isWhitespace(character) && quote == 0) { + if (!current.isEmpty()) { + tokens.add(current.toString()); + current.setLength(0); + } + } else { + current.append(character); + } + } + if (escaping || quote != 0) { + throw new AgentRuntimeException("Shell command contains an unfinished escape or quote."); + } + if (!current.isEmpty()) { + tokens.add(current.toString()); + } + if (tokens.isEmpty()) { + throw new AgentRuntimeException("Shell command is required."); + } + return tokens; + } + + private void validate(List arguments) { + String executable = arguments.get(0); + if (executable.contains("/") || executable.contains("\\") || !allowedCommands.contains(executable)) { + throw new AgentRuntimeException("Shell command is not allowlisted: " + executable); + } + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + rejectHostOrTraversalPath(argument); + validateExistingPathArgument(argument); + } + optionValidator.validate(arguments); + if ("python".equals(executable) || "python3".equals(executable)) { + validateScript(arguments, Set.of(".py"), "-c", "-m"); + } else if ("node".equals(executable)) { + validateScript(arguments, Set.of(".js", ".mjs", ".cjs"), "-e", "--eval"); + } else if ("rm".equals(executable)) { + validateRemove(arguments); + } + } + + private void validateScript(List arguments, Set extensions, String... deniedOptions) { + if (arguments.size() < 2 || arguments.get(1).startsWith("-")) { + throw new AgentRuntimeException("Script command requires a workspace script file as its first argument."); + } + for (String denied : deniedOptions) { + if (arguments.contains(denied)) { + throw new AgentRuntimeException("Inline or module script execution is not allowed."); + } + } + String script = arguments.get(1); + if (extensions.stream().noneMatch(script::endsWith)) { + throw new AgentRuntimeException("Script file extension is not allowed."); + } + pathGuard.resolveExistingFile(script); + } + + private void validateRemove(List arguments) { + boolean hasTarget = false; + boolean recursive = false; + boolean force = false; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if (argument.startsWith("-")) { + String flags = argument.replace("-", ""); + recursive |= flags.contains("r") || flags.contains("R") || "recursive".equals(flags); + force |= flags.contains("f") || "force".equals(flags); + continue; + } + if (".".equals(argument) || "./".equals(argument)) { + throw new AgentRuntimeException("Workspace root cannot be removed."); + } + hasTarget = true; + } + if (!hasTarget) { + throw new AgentRuntimeException("rm requires at least one workspace target."); + } + if (recursive && force) { + throw new AgentRuntimeException("Recursive forced removal is not allowed."); + } + } + + private void rejectHostOrTraversalPath(String argument) { + if (argument.startsWith("-") + && (argument.contains("/") || argument.contains("\\") || argument.contains("~"))) { + throw new AgentRuntimeException("Shell option-embedded paths are not allowed."); + } + String candidate = optionValue(argument); + if (candidate.isEmpty() || candidate.startsWith("-")) { + return; + } + if (candidate.startsWith("/") || candidate.startsWith("\\") + || candidate.matches("^[A-Za-z]:[\\\\/].*") || candidate.startsWith("~")) { + throw new AgentRuntimeException("Shell absolute paths are not allowed."); + } + if (candidate.matches("^[A-Za-z][A-Za-z0-9+.-]*://.*") + || candidate.regionMatches(true, 0, "file:", 0, "file:".length()) + || candidate.regionMatches(true, 0, "data:", 0, "data:".length())) { + throw new AgentRuntimeException("Shell URI inputs are not allowed."); + } + for (String segment : candidate.replace('\\', '/').split("/")) { + if ("..".equals(segment)) { + throw new AgentRuntimeException("Shell path traversal is not allowed."); + } + } + } + + private void validateExistingPathArgument(String argument) { + String candidate = optionValue(argument); + if (candidate.isEmpty() || candidate.startsWith("-") || candidate.equals(".")) { + return; + } + Path possible = pathGuard.root().resolve(candidate).normalize(); + if (!possible.startsWith(pathGuard.root()) || !Files.exists(possible, LinkOption.NOFOLLOW_LINKS)) { + return; + } + pathGuard.resolveExistingEntry(candidate); + } + + private String optionValue(String argument) { + int equals = argument.indexOf('='); + return equals >= 0 ? argument.substring(equals + 1) : argument; + } + + private void sanitizeEnvironment(Map environment) { + environment.clear(); + // 固定搜索路径,避免宿主继承 PATH 中的可写目录劫持白名单命令。 + environment.put("PATH", TRUSTED_EXECUTABLE_PATH); + environment.put("PYTHONPATH", "/opt/easyflow/python-packages"); + environment.put("NODE_PATH", "/app/node_modules"); + environment.put("HOME", pathGuard.root().toString()); + environment.put("TMPDIR", pathGuard.root().toString()); + environment.put("LANG", "C.UTF-8"); + environment.put("LC_ALL", "C.UTF-8"); + } + + private String requiredCommand(ToolCallParam param) { + Object value = param == null ? null : param.getInput().get("command"); + return requiredCommand(value); + } + + /** + * 从动态审批入参中读取命令。 + * + * @param input 工具调用入参 + * @return 已完成基础校验的命令 + */ + private String requiredCommand(Map input) { + Object value = input == null ? null : input.get("command"); + return requiredCommand(value); + } + + /** + * 校验命令值与最大长度。 + * + * @param value 原始命令值 + * @return 已完成基础校验的命令 + */ + private String requiredCommand(Object value) { + if (!(value instanceof String command) || command.isBlank()) { + throw new AgentRuntimeException("Shell command is required."); + } + if (command.length() > maxCommandLength) { + throw new AgentRuntimeException("Shell command exceeds max-command-length."); + } + return command; + } + + /** + * 根据脚本内容和完整参数计算当前 Turn 的复用审批作用域。 + * + * @param arguments 命令参数 + * @return 带类型前缀的 SHA-256 审批作用域 + */ + private String scriptApprovalScope(List arguments) { + Path script = pathGuard.resolveExistingFile(arguments.get(1)); + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + try (InputStream input = Files.newInputStream(script)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) >= 0) { + digest.update(buffer, 0, read); + } + } + for (String argument : arguments) { + digest.update((byte) 0); + digest.update(argument.getBytes(StandardCharsets.UTF_8)); + } + return "SHELL_SCRIPT:" + java.util.HexFormat.of().formatHex(digest.digest()); + } catch (IOException error) { + throw new WorkspaceToolException( + "WORKSPACE_IO_FAILED", "Script could not be hashed before approval.", true, error); + } catch (NoSuchAlgorithmException error) { + throw new AgentRuntimeException("SHA-256 is unavailable for script approval.", error); + } + } + + private int requestedTimeout(ToolCallParam param) { + Object value = param == null ? null : param.getInput().get("timeout"); + if (value == null) { + return defaultTimeoutSeconds; + } + if (!(value instanceof Number number)) { + throw new AgentRuntimeException("Shell timeout must be an integer number of seconds."); + } + int timeout = number.intValue(); + if (timeout <= 0 || timeout > maxTimeoutSeconds) { + throw new AgentRuntimeException("Shell timeout is outside the configured range."); + } + return timeout; + } + + private void validateCharset(ToolCallParam param) { + Object value = param == null ? null : param.getInput().get("charset"); + if (value != null && (!(value instanceof String charset) || !"UTF-8".equalsIgnoreCase(charset.trim()))) { + throw new AgentRuntimeException("Shell charset override is limited to UTF-8."); + } + } + + private CompletableFuture readBounded(InputStream input) { + try { + return CompletableFuture.supplyAsync(() -> { + ByteArrayOutputStream retained = new ByteArrayOutputStream(Math.min(maxOutputSize, 8192)); + boolean truncated = false; + byte[] buffer = new byte[8192]; + try (input) { + int read; + while ((read = input.read(buffer)) >= 0) { + int remaining = maxOutputSize - retained.size(); + if (remaining > 0) { + retained.write(buffer, 0, Math.min(read, remaining)); + } + if (read > remaining) { + truncated = true; + } + } + } catch (IOException error) { + throw new WorkspaceToolException("SHELL_OUTPUT_FAILED", + "Shell output stream could not be read.", true, error); + } + return new BoundedOutput(retained.toString(StandardCharsets.UTF_8), truncated); + }, outputExecutor); + } catch (RejectedExecutionException error) { + throw new WorkspaceToolException("SHELL_CONCURRENCY_LIMIT", + "Shell output collector is at capacity.", true, error); + } + } + + private BoundedOutput awaitOutput(CompletableFuture future) { + try { + return future.get(2, TimeUnit.SECONDS); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new WorkspaceToolException("SHELL_INTERRUPTED", + "Shell output collection was interrupted.", true, error); + } catch (ExecutionException | java.util.concurrent.TimeoutException error) { + future.cancel(true); + Throwable cause = error instanceof ExecutionException && error.getCause() != null + ? error.getCause() : error; + if (cause instanceof WorkspaceToolException typed) { + throw typed; + } + throw new WorkspaceToolException("SHELL_OUTPUT_FAILED", + "Shell output could not be collected.", true, cause); + } + } + + private boolean waitForProcess(Process process, + int timeoutSeconds, + Set observedDescendants) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (process.isAlive()) { + observedDescendants.addAll(process.toHandle().descendants().toList()); + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()); + if (remainingMillis <= 0) { + return false; + } + process.waitFor(Math.max(1, Math.min(remainingMillis, 10)), TimeUnit.MILLISECONDS); + } + observedDescendants.addAll(process.toHandle().descendants().toList()); + return true; + } + + private void terminateProcessTree(Process process, + Set observedDescendants, + long processGroupId) { + processGroupSupport.terminate(processGroupId); + List descendants = new ArrayList<>(observedDescendants); + descendants.addAll(process.toHandle().descendants().toList()); + for (int index = descendants.size() - 1; index >= 0; index--) { + descendants.get(index).destroy(); + } + process.destroy(); + try { + if (!process.waitFor(500, TimeUnit.MILLISECONDS)) { + for (int index = descendants.size() - 1; index >= 0; index--) { + ProcessHandle descendant = descendants.get(index); + if (descendant.isAlive()) { + descendant.destroyForcibly(); + } + } + process.destroyForcibly(); + process.waitFor(500, TimeUnit.MILLISECONDS); + } + } catch (InterruptedException error) { + for (ProcessHandle descendant : descendants) { + if (descendant.isAlive()) { + descendant.destroyForcibly(); + } + } + process.destroyForcibly(); + Thread.currentThread().interrupt(); + } + } + + private void terminateObservedDescendants(Set observedDescendants) { + Set expanded = new LinkedHashSet<>(observedDescendants); + for (ProcessHandle descendant : observedDescendants) { + if (descendant.isAlive()) { + expanded.addAll(descendant.descendants().toList()); + } + } + for (ProcessHandle descendant : expanded) { + if (descendant.isAlive()) { + descendant.destroy(); + } + } + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(300); + while (expanded.stream().anyMatch(ProcessHandle::isAlive) + && System.nanoTime() < deadline) { + try { + Thread.sleep(10); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + break; + } + } + for (ProcessHandle descendant : expanded) { + if (descendant.isAlive()) { + descendant.destroyForcibly(); + } + } + } + + private static void terminateProcessTreeNow(Process process, Set observedDescendants) { + if (process == null) { + return; + } + List descendants = new ArrayList<>(observedDescendants); + descendants.addAll(process.toHandle().descendants().toList()); + for (int index = descendants.size() - 1; index >= 0; index--) { + ProcessHandle descendant = descendants.get(index); + if (descendant.isAlive()) { + descendant.destroyForcibly(); + } + } + if (process.isAlive()) { + process.destroyForcibly(); + } + } + + private ToolResultBlock result(int returnCode, + BoundedOutput stdout, + BoundedOutput stderr, + String errorCode, + String errorMessage, + boolean retryable, + long durationMillis) { + String error = errorCode == null ? "" : "" + errorCode + "" + + xml(errorMessage) + "" + retryable + ""; + String warning = errorCode == null && (stdout.truncated() || stderr.truncated()) + ? "OUTPUT_TRUNCATEDShell output exceeded the configured limit." + + "false" : ""; + String formatted = "" + returnCode + "" + + "" + xml(sanitizeOutput(stdout.text())) + "" + + "" + xml(sanitizeOutput(stderr.text())) + "" + + "" + durationMillis + "" + error + warning; + return ToolResultBlock.text(formatted); + } + + private String xml(String value) { + if (value == null) { + return ""; + } + return value.replace("&", "&").replace("<", "<").replace(">", ">"); + } + + private String sanitizeOutput(String value) { + if (value == null || value.isEmpty()) { + return ""; + } + return value.replace(pathGuard.root().toString(), "."); + } + + private static int seconds(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative() || duration.getSeconds() > Integer.MAX_VALUE) { + throw new AgentRuntimeException(name + " must be a positive whole-second duration."); + } + return Math.toIntExact(duration.getSeconds()); + } + + private static Set validateAllowedCommands(Set configured) { + if (configured == null || configured.isEmpty()) { + throw new AgentRuntimeException("Shell command whitelist must not be empty."); + } + Set normalized = new LinkedHashSet<>(); + for (String command : configured) { + if (command == null || command.isBlank() || !DEFAULT_ALLOWED_COMMANDS.contains(command.trim())) { + throw new AgentRuntimeException("Shell command is outside the fixed whitelist."); + } + normalized.add(command.trim()); + } + return Set.copyOf(normalized); + } + + private static ExecutorService createOutputExecutor(int maxConcurrency) { + int threads = Math.multiplyExact(maxConcurrency, 2); + ThreadFactory threadFactory = runnable -> { + Thread thread = new Thread(runnable, + "easyagents-shell-output-" + OUTPUT_THREAD_SEQUENCE.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + return new ThreadPoolExecutor( + threads, + threads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(Math.max(threads * 2, 4)), + threadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + /** + * 有界输出。 + * + * @param text 保留文本 + * @param truncated 是否截断 + */ + private record BoundedOutput(String text, boolean truncated) { + } + + /** + * 活跃命令及其进程组清理上下文。 + * + * @param observedDescendants 执行期观察到的后代 + * @param processGroupSupport Linux 进程组支持 + * @param processGroupId Linux PGID,降级模式为 -1 + */ + private record ActiveProcess(Set observedDescendants, + ShellProcessGroupSupport processGroupSupport, + long processGroupId) { + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutor.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutor.java new file mode 100644 index 0000000..b9201bd --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutor.java @@ -0,0 +1,1319 @@ +package com.easyagents.agent.runtime.tool.operate; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.UnixStat; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import java.util.zip.CRC32; + +/** + * 使用纯 Java 流完成的工作区安全归档命令执行器。 + * + *

所有归档输入先复制到工作区外、同文件系统的临时目录。解包会先完整扫描条目,再写入 + * staging,最后以逐文件原子移动和失败补偿提交,系统归档程序不会参与执行。 + */ +final class SafeArchiveCommandExecutor { + + private static final Logger logger = LoggerFactory.getLogger(SafeArchiveCommandExecutor.class); + static final Set COMMANDS = Set.of("gzip", "gunzip", "zip", "unzip", "tar"); + + private static final Pattern WINDOWS_ABSOLUTE = Pattern.compile("^[A-Za-z]:/.*"); + private static final int MAX_PATH_LENGTH = 512; + private static final int MAX_PATH_DEPTH = 64; + private static final int COPY_BUFFER_SIZE = 16 * 1024; + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + private final int maxOutputBytes; + private final ArchiveCommitObserver commitObserver; + + /** + * 创建归档命令执行器。 + * + * @param pathGuard 工作区路径保护器 + * @param quotaGuard 工作区配额保护器 + * @param maxOutputBytes 命令文本输出最大字节数 + */ + SafeArchiveCommandExecutor(WorkspacePathGuard pathGuard, + WorkspaceQuotaGuard quotaGuard, + int maxOutputBytes) { + this(pathGuard, quotaGuard, maxOutputBytes, ArchiveCommitObserver.noop()); + } + + /** + * 创建带提交观察器的归档执行器,供事务补偿测试使用。 + * + * @param pathGuard 工作区路径保护器 + * @param quotaGuard 工作区配额保护器 + * @param maxOutputBytes 命令文本输出最大字节数 + * @param commitObserver 提交观察器 + */ + SafeArchiveCommandExecutor(WorkspacePathGuard pathGuard, + WorkspaceQuotaGuard quotaGuard, + int maxOutputBytes, + ArchiveCommitObserver commitObserver) { + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + this.maxOutputBytes = maxOutputBytes; + this.commitObserver = commitObserver == null ? ArchiveCommitObserver.noop() : commitObserver; + } + + /** + * 执行已经完成 Shell 安全分词的归档命令。 + * + * @param arguments 命令及参数 + * @param deadlineNanos 单调时钟截止时间 + * @return 有界文本结果 + */ + ArchiveExecutionResult execute(List arguments, long deadlineNanos) { + if (arguments == null || arguments.isEmpty() || !COMMANDS.contains(arguments.get(0))) { + throw denied("Unsupported archive command."); + } + try (Stage stage = Stage.create(pathGuard.root())) { + return switch (arguments.get(0)) { + case "gzip" -> gzip(arguments, false, stage, deadlineNanos); + case "gunzip" -> gzip(arguments, true, stage, deadlineNanos); + case "zip" -> zip(arguments, stage, deadlineNanos); + case "unzip" -> unzip(arguments, stage, deadlineNanos); + case "tar" -> tar(arguments, stage, deadlineNanos); + default -> throw denied("Unsupported archive command."); + }; + } catch (WorkspaceToolException error) { + throw error; + } catch (IOException error) { + throw new WorkspaceToolException("ARCHIVE_IO_FAILED", + "Archive operation failed.", true, error); + } + } + + private ArchiveExecutionResult gzip(List arguments, + boolean decompress, + Stage stage, + long deadlineNanos) throws IOException { + ParsedFileList parsed = parseFileList(arguments, Set.of("-k")); + boolean keep = parsed.options().contains("-k"); + if (parsed.files().isEmpty()) { + throw denied(arguments.get(0) + " requires at least one workspace file."); + } + List outputs = new ArrayList<>(); + List deletions = new ArrayList<>(); + Map quotaPlan = new LinkedHashMap<>(); + OutputCollector output = new OutputCollector(maxOutputBytes); + long expandedTotal = 0; + for (String file : parsed.files()) { + checkDeadline(deadlineNanos); + Path input = pathGuard.resolveExistingFile(file); + String targetName = decompress ? gunzipTarget(file) : file + ".gz"; + Path target = newOutputTarget(targetName); + Snapshot snapshot = snapshotFile(input, stage, deadlineNanos); + Path stagedOutput = stage.newFile("gzip-output-"); + if (decompress) { + long written; + try (InputStream raw = Files.newInputStream(snapshot.path(), StandardOpenOption.READ); + GzipCompressorInputStream gzip = new GzipCompressorInputStream(raw); + OutputStream targetOutput = limitedFileOutput( + stagedOutput, quotaGuard.maxArchiveSingleFileSize())) { + written = transfer(gzip, targetOutput, + quotaGuard.maxArchiveSingleFileSize(), deadlineNanos, null); + } catch (IOException error) { + throw invalidArchive("GZIP input is invalid.", error); + } + expandedTotal = addExpanded(expandedTotal, written); + } else { + GzipParameters parameters = new GzipParameters(); + parameters.setModificationInstant(Instant.EPOCH); + try (OutputStream raw = limitedFileOutput( + stagedOutput, quotaGuard.maxArchiveSingleFileSize()); + GzipCompressorOutputStream gzip = new GzipCompressorOutputStream(raw, parameters); + InputStream source = Files.newInputStream(snapshot.path(), StandardOpenOption.READ)) { + transfer(source, gzip, snapshot.size(), deadlineNanos, null); + } + expandedTotal = addExpanded(expandedTotal, snapshot.size()); + } + forceFile(stagedOutput); + long outputSize = Files.size(stagedOutput); + outputs.add(new OutputPlan(target, stagedOutput)); + quotaPlan.put(target, outputSize); + if (!keep) { + deletions.add(new DeletionPlan(input, snapshot.fingerprint())); + quotaPlan.put(input, -1L); + } + output.append(pathGuard.display(target)).append('\n'); + } + quotaGuard.validateBatch(quotaPlan); + commit(stage, outputs, List.of(), deletions, deadlineNanos); + return output.result(); + } + + private ArchiveExecutionResult zip(List arguments, + Stage stage, + long deadlineNanos) throws IOException { + ParsedZip parsed = parseZip(arguments); + Path target = newOutputTarget(parsed.archive()); + List sources = snapshotSources( + parsed.inputs(), pathGuard.root(), parsed.recursive(), stage, deadlineNanos); + Path stagedArchive = stage.newFile("zip-output-"); + try (OutputStream raw = limitedFileOutput(stagedArchive, quotaGuard.maxArchiveSingleFileSize()); + ZipArchiveOutputStream zip = new ZipArchiveOutputStream(raw)) { + zip.setEncoding(StandardCharsets.UTF_8.name()); + zip.setUseLanguageEncodingFlag(true); + for (ArchiveSource source : sources) { + checkDeadline(deadlineNanos); + String name = source.directory() ? source.name() + "/" : source.name(); + ZipArchiveEntry entry = new ZipArchiveEntry(name); + entry.setTime(0L); + entry.setUnixMode(source.directory() ? UnixStat.DEFAULT_DIR_PERM : UnixStat.DEFAULT_FILE_PERM); + zip.putArchiveEntry(entry); + if (!source.directory()) { + try (InputStream input = Files.newInputStream(source.snapshot(), StandardOpenOption.READ)) { + transfer(input, zip, source.size(), deadlineNanos, null); + } + } + zip.closeArchiveEntry(); + } + zip.finish(); + } + forceFile(stagedArchive); + quotaGuard.validateWrite(target, Files.size(stagedArchive)); + commit(stage, List.of(new OutputPlan(target, stagedArchive)), List.of(), List.of(), deadlineNanos); + return textResult(pathGuard.display(target) + "\n"); + } + + private ArchiveExecutionResult unzip(List arguments, + Stage stage, + long deadlineNanos) throws IOException { + ParsedExtract parsed = parseUnzip(arguments); + Path archive = pathGuard.resolveExistingFile(parsed.archive()); + Snapshot snapshot = snapshotFile(archive, stage, deadlineNanos); + enforceZipCentralDirectoryCount(snapshot.path()); + Path destination = resolveDestination(parsed.destination()); + List plans = scanZip(snapshot.path(), destination, deadlineNanos); + return extractZip(snapshot.path(), plans, stage, deadlineNanos); + } + + private ArchiveExecutionResult tar(List arguments, + Stage stage, + long deadlineNanos) throws IOException { + ParsedTar parsed = parseTar(arguments); + if (parsed.mode().create()) { + Path target = newOutputTarget(parsed.archive()); + Path base = parsed.destination() == null + ? pathGuard.root() : pathGuard.resolveExistingDirectory(parsed.destination()); + List sources = snapshotSources( + parsed.inputs(), base, true, stage, deadlineNanos); + Path stagedArchive = stage.newFile("tar-output-"); + writeTar(stagedArchive, sources, parsed.mode().gzip(), deadlineNanos); + quotaGuard.validateWrite(target, Files.size(stagedArchive)); + commit(stage, List.of(new OutputPlan(target, stagedArchive)), List.of(), List.of(), deadlineNanos); + return textResult(pathGuard.display(target) + "\n"); + } + Path archive = pathGuard.resolveExistingFile(parsed.archive()); + Snapshot snapshot = snapshotFile(archive, stage, deadlineNanos); + Path destination = parsed.destination() == null + ? pathGuard.root() : resolveDestination(parsed.destination()); + List plans = scanTar( + snapshot.path(), destination, parsed.mode().gzip(), parsed.mode().list(), deadlineNanos); + if (parsed.mode().list()) { + OutputCollector output = new OutputCollector(maxOutputBytes); + for (EntryPlan plan : plans) { + output.append(plan.name()).append(plan.directory() ? "/\n" : "\n"); + } + return output.result(); + } + return extractTar(snapshot.path(), plans, parsed.mode().gzip(), stage, deadlineNanos); + } + + private List snapshotSources(List inputs, + Path base, + boolean recursive, + Stage stage, + long deadlineNanos) throws IOException { + if (inputs.isEmpty()) { + throw denied("Archive creation requires at least one input."); + } + EntryIndex index = new EntryIndex(); + List sources = new ArrayList<>(); + long total = 0; + for (String inputValue : inputs) { + checkDeadline(deadlineNanos); + Path input = resolveInput(base, inputValue); + String rootName = normalizeEntryName(inputValue); + if (Files.isDirectory(input, LinkOption.NOFOLLOW_LINKS)) { + if (!recursive) { + throw denied("Directory inputs require the recursive option."); + } + try (Stream paths = Files.walk(input)) { + for (Path source : (Iterable) paths::iterator) { + checkDeadline(deadlineNanos); + String relative = input.equals(source) ? "" + : input.relativize(source).toString().replace(source.getFileSystem().getSeparator(), "/"); + String name = relative.isEmpty() ? rootName : rootName + "/" + relative; + if (Files.isSymbolicLink(source)) { + throw entryDenied("Archive input contains a symbolic link."); + } + if (Files.isDirectory(source, LinkOption.NOFOLLOW_LINKS)) { + index.add(name, true); + sources.add(new ArchiveSource(name, null, true, 0)); + } else if (Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS)) { + Path guarded = pathGuard.resolveExistingFile(pathGuard.display(source)); + Snapshot snapshot = snapshotFile(guarded, stage, deadlineNanos); + total = addExpanded(total, snapshot.size()); + index.add(name, false); + sources.add(new ArchiveSource(name, snapshot.path(), false, snapshot.size())); + } else { + throw entryDenied("Archive input contains a non-regular entry."); + } + ensureEntryCount(sources.size()); + } + } + } else { + Path guarded = pathGuard.resolveExistingFile(pathGuard.display(input)); + Snapshot snapshot = snapshotFile(guarded, stage, deadlineNanos); + total = addExpanded(total, snapshot.size()); + index.add(rootName, false); + sources.add(new ArchiveSource(rootName, snapshot.path(), false, snapshot.size())); + ensureEntryCount(sources.size()); + } + } + return sources; + } + + private Path resolveInput(Path base, String inputValue) { + String normalized = normalizeEntryName(inputValue); + String baseName = pathGuard.display(base); + String combined = ".".equals(baseName) ? normalized : baseName + "/" + normalized; + return pathGuard.resolveExistingEntry(combined); + } + + private Snapshot snapshotFile(Path source, Stage stage, long deadlineNanos) throws IOException { + pathGuard.revalidate(source); + BasicFileAttributes before = Files.readAttributes( + source, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!before.isRegularFile()) { + throw entryDenied("Archive input is not a regular file."); + } + if (before.size() > quotaGuard.maxArchiveSingleFileSize()) { + throw limitExceeded("Archive input exceeds the single-file limit."); + } + Fingerprint fingerprint = Fingerprint.from(before); + Path snapshot = stage.newFile("archive-input-"); + Set inputOptions = Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS); + try (SeekableByteChannel input = Files.newByteChannel(source, inputOptions); + InputStream inputStream = java.nio.channels.Channels.newInputStream(input); + OutputStream output = limitedFileOutput(snapshot, quotaGuard.maxArchiveSingleFileSize())) { + transfer(inputStream, output, before.size(), deadlineNanos, null); + } + forceFile(snapshot); + pathGuard.revalidate(source); + BasicFileAttributes after = Files.readAttributes( + source, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!fingerprint.matches(after) || Files.size(snapshot) != before.size()) { + throw new WorkspaceToolException("ARCHIVE_INPUT_CHANGED", + "Archive input changed while it was being copied.", true); + } + return new Snapshot(snapshot, before.size(), fingerprint); + } + + private void writeTar(Path target, + List sources, + boolean gzip, + long deadlineNanos) throws IOException { + try (OutputStream limited = limitedFileOutput(target, quotaGuard.maxArchiveSingleFileSize()); + OutputStream compressed = gzip ? newGzipOutput(limited) : limited; + TarArchiveOutputStream tar = new TarArchiveOutputStream(compressed, StandardCharsets.UTF_8.name())) { + tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_ERROR); + for (ArchiveSource source : sources) { + checkDeadline(deadlineNanos); + TarArchiveEntry entry = new TarArchiveEntry( + source.directory() ? source.name() + "/" : source.name()); + entry.setMode(source.directory() ? TarArchiveEntry.DEFAULT_DIR_MODE : TarArchiveEntry.DEFAULT_FILE_MODE); + entry.setModTime(0L); + entry.setSize(source.directory() ? 0 : source.size()); + tar.putArchiveEntry(entry); + if (!source.directory()) { + try (InputStream input = Files.newInputStream(source.snapshot(), StandardOpenOption.READ)) { + transfer(input, tar, source.size(), deadlineNanos, null); + } + } + tar.closeArchiveEntry(); + } + tar.finish(); + } + forceFile(target); + } + + private List scanZip(Path archive, + Path destination, + long deadlineNanos) { + List plans = new ArrayList<>(); + EntryIndex index = new EntryIndex(); + long total = 0; + try (ZipFile zip = new ZipFile(archive)) { + Enumeration entries = zip.getEntriesInPhysicalOrder(); + while (entries.hasMoreElements()) { + checkDeadline(deadlineNanos); + ZipArchiveEntry entry = entries.nextElement(); + String name = strictZipEntryName(entry); + boolean directory = entry.isDirectory(); + validateZipType(zip, entry); + index.add(name, directory); + ensureEntryCount(totalEntryCounter(plans)); + long size = directory ? 0 : entry.getSize(); + long compressed = directory ? 0 : entry.getCompressedSize(); + if (size < 0 || compressed < 0 || (!directory && entry.getCrc() < 0)) { + throw invalidArchive("ZIP entry metadata is incomplete.", null); + } + if (!directory) { + ensureSingleSize(size); + total = addExpanded(total, size); + } + plans.add(entryPlan(destination, name, directory, size, entry.getCrc())); + } + } catch (WorkspaceToolException error) { + throw error; + } catch (IOException error) { + throw invalidArchive("ZIP archive is invalid.", error); + } + return plans; + } + + private ArchiveExecutionResult extractZip(Path archive, + List plans, + Stage stage, + long deadlineNanos) throws IOException { + List outputs = new ArrayList<>(); + List directories = new ArrayList<>(); + Map quotaPlan = quotaExtractionPlan(plans); + OutputCollector result = new OutputCollector(maxOutputBytes); + try (ZipFile zip = new ZipFile(archive)) { + for (EntryPlan plan : plans) { + checkDeadline(deadlineNanos); + if (plan.directory()) { + directories.add(plan.target()); + result.append(plan.name()).append("/\n"); + continue; + } + ZipArchiveEntry entry = zip.getEntry(plan.name()); + if (entry == null) { + throw invalidArchive("ZIP entry disappeared between validation and extraction.", null); + } + Path staged = stage.newFile("zip-entry-"); + CRC32 crc = new CRC32(); + try (InputStream input = zip.getInputStream(entry); + OutputStream output = limitedFileOutput(staged, quotaGuard.maxArchiveSingleFileSize())) { + long written = transfer(input, output, plan.size(), deadlineNanos, crc); + if (written != plan.size() || crc.getValue() != plan.crc()) { + throw invalidArchive("ZIP entry size or CRC does not match metadata.", null); + } + } + forceFile(staged); + outputs.add(new OutputPlan(plan.target(), staged)); + result.append(plan.name()).append('\n'); + } + } catch (WorkspaceToolException error) { + throw error; + } catch (IOException error) { + throw invalidArchive("ZIP archive data is invalid.", error); + } + quotaGuard.validateBatch(quotaPlan); + commit(stage, outputs, directories, List.of(), deadlineNanos); + return result.result(); + } + + private List scanTar(Path archive, + Path destination, + boolean gzip, + boolean listOnly, + long deadlineNanos) { + List plans = new ArrayList<>(); + EntryIndex index = new EntryIndex(); + long total = 0; + try (TarArchiveInputStream tar = openTar(archive, gzip)) { + TarArchiveEntry entry; + while ((entry = tar.getNextTarEntry()) != null) { + checkDeadline(deadlineNanos); + validateTarType(entry); + String name = normalizeEntryName(entry.getName()); + boolean directory = entry.isDirectory(); + index.add(name, directory); + ensureEntryCount(plans.size() + 1); + long size = directory ? 0 : entry.getSize(); + if (!directory) { + ensureSingleSize(size); + total = addExpanded(total, size); + } + plans.add(listOnly + ? new EntryPlan(name, null, directory, size, -1) + : entryPlan(destination, name, directory, size, -1)); + } + } catch (WorkspaceToolException error) { + throw error; + } catch (IOException error) { + throw invalidArchive("TAR archive is invalid.", error); + } + return plans; + } + + private ArchiveExecutionResult extractTar(Path archive, + List plans, + boolean gzip, + Stage stage, + long deadlineNanos) throws IOException { + List outputs = new ArrayList<>(); + List directories = new ArrayList<>(); + Map quotaPlan = quotaExtractionPlan(plans); + OutputCollector result = new OutputCollector(maxOutputBytes); + int index = 0; + try (TarArchiveInputStream tar = openTar(archive, gzip)) { + TarArchiveEntry entry; + while ((entry = tar.getNextTarEntry()) != null) { + checkDeadline(deadlineNanos); + EntryPlan plan = plans.get(index++); + if (plan.directory()) { + directories.add(plan.target()); + result.append(plan.name()).append("/\n"); + continue; + } + Path staged = stage.newFile("tar-entry-"); + try (OutputStream output = limitedFileOutput(staged, quotaGuard.maxArchiveSingleFileSize())) { + long written = transfer(tar, output, plan.size(), deadlineNanos, null); + if (written != plan.size()) { + throw invalidArchive("TAR entry size does not match its header.", null); + } + } + forceFile(staged); + outputs.add(new OutputPlan(plan.target(), staged)); + result.append(plan.name()).append('\n'); + } + } catch (WorkspaceToolException error) { + throw error; + } catch (IOException | IndexOutOfBoundsException error) { + throw invalidArchive("TAR archive data is invalid.", error); + } + quotaGuard.validateBatch(quotaPlan); + commit(stage, outputs, directories, List.of(), deadlineNanos); + return result.result(); + } + + private Map quotaExtractionPlan(List plans) { + Map quotaPlan = new LinkedHashMap<>(); + for (EntryPlan plan : plans) { + if (plan.directory()) { + if (!Files.exists(plan.target(), LinkOption.NOFOLLOW_LINKS)) { + quotaPlan.put(plan.target(), 0L); + } + } else { + quotaPlan.put(plan.target(), plan.size()); + } + } + return quotaPlan; + } + + private EntryPlan entryPlan(Path destination, + String name, + boolean directory, + long size, + long crc) { + String destinationName = pathGuard.display(destination); + String combined = ".".equals(destinationName) ? name : destinationName + "/" + name; + Path target = directory ? pathGuard.resolveCommandPath(combined) : pathGuard.resolveForWrite(combined); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + if (directory && Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS)) { + return new EntryPlan(name, target, true, 0, crc); + } + throw new WorkspaceToolException("ARCHIVE_TARGET_CONFLICT", + "Archive output target already exists.", false); + } + return new EntryPlan(name, target, directory, size, crc); + } + + private void commit(Stage stage, + List outputs, + List directories, + List deletions, + long deadlineNanos) { + List createdDirectories = new ArrayList<>(); + List committedOutputs = new ArrayList<>(); + Map deletionBackups = new LinkedHashMap<>(); + try { + checkDeadline(deadlineNanos); + for (OutputPlan output : outputs) { + pathGuard.revalidate(output.target()); + if (Files.exists(output.target(), LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("ARCHIVE_TARGET_CONFLICT", + "Archive output target already exists.", false); + } + } + for (DeletionPlan deletion : deletions) { + verifyFingerprint(deletion.target(), deletion.fingerprint()); + } + for (Path directory : directories.stream().sorted(Comparator.comparingInt(Path::getNameCount)).toList()) { + ensureDirectory(directory, createdDirectories); + } + for (OutputPlan output : outputs) { + ensureDirectory(output.target().getParent(), createdDirectories); + } + for (DeletionPlan deletion : deletions) { + Path backup = stage.newFile("archive-backup-"); + Files.deleteIfExists(backup); + atomicMove(deletion.target(), backup); + deletionBackups.put(deletion.target(), backup); + } + for (OutputPlan output : outputs) { + checkDeadline(deadlineNanos); + atomicMove(output.staged(), output.target()); + committedOutputs.add(output.target()); + commitObserver.afterOutputCommitted(committedOutputs.size()); + forceDirectory(output.target().getParent()); + } + } catch (Exception error) { + Exception rollbackError = rollback(committedOutputs, deletionBackups, createdDirectories); + if (rollbackError != null) { + error.addSuppressed(rollbackError); + throw new WorkspaceToolException("ARCHIVE_ROLLBACK_FAILED", + "Archive commit and rollback failed; workspace requires inspection.", false, error); + } + if (error instanceof WorkspaceToolException typed) { + throw typed; + } + throw new WorkspaceToolException("ARCHIVE_COMMIT_FAILED", + "Archive commit failed and changes were rolled back.", true, error); + } + } + + private Exception rollback(List committedOutputs, + Map deletionBackups, + List createdDirectories) { + Exception failure = null; + for (int index = committedOutputs.size() - 1; index >= 0; index--) { + try { + Files.deleteIfExists(committedOutputs.get(index)); + } catch (IOException error) { + failure = appendFailure(failure, error); + } + } + List> backups = new ArrayList<>(deletionBackups.entrySet()); + for (int index = backups.size() - 1; index >= 0; index--) { + try { + atomicMove(backups.get(index).getValue(), backups.get(index).getKey()); + } catch (IOException error) { + failure = appendFailure(failure, error); + } + } + for (int index = createdDirectories.size() - 1; index >= 0; index--) { + try { + Files.deleteIfExists(createdDirectories.get(index)); + } catch (IOException error) { + failure = appendFailure(failure, error); + } + } + return failure; + } + + private Exception appendFailure(Exception current, Exception next) { + if (current == null) { + return next; + } + current.addSuppressed(next); + return current; + } + + private void ensureDirectory(Path directory, List created) throws IOException { + if (directory == null || directory.equals(pathGuard.root())) { + return; + } + Path current = pathGuard.root(); + for (Path segment : pathGuard.root().relativize(directory)) { + current = current.resolve(segment); + if (Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { + if (!Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(current)) { + throw new WorkspaceToolException("ARCHIVE_TARGET_CONFLICT", + "Archive output parent conflicts with an existing entry.", false); + } + } else { + Files.createDirectory(current); + created.add(current); + } + } + } + + private void verifyFingerprint(Path target, Fingerprint expected) throws IOException { + pathGuard.revalidate(target); + BasicFileAttributes actual = Files.readAttributes( + target, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!expected.matches(actual)) { + throw new WorkspaceToolException("ARCHIVE_INPUT_CHANGED", + "Archive input changed before commit.", true); + } + } + + private TarArchiveInputStream openTar(Path archive, boolean gzip) throws IOException { + InputStream raw = Files.newInputStream(archive, StandardOpenOption.READ); + try { + InputStream input = gzip ? new GzipCompressorInputStream(raw) : raw; + return new TarArchiveInputStream(input, StandardCharsets.UTF_8.name()); + } catch (IOException error) { + raw.close(); + throw error; + } + } + + private OutputStream newGzipOutput(OutputStream output) throws IOException { + GzipParameters parameters = new GzipParameters(); + parameters.setModificationInstant(Instant.EPOCH); + return new GzipCompressorOutputStream(output, parameters); + } + + private void validateZipType(ZipFile zip, ZipArchiveEntry entry) { + if (!zip.canReadEntryData(entry)) { + throw entryDenied("Encrypted or unsupported ZIP entries are not allowed."); + } + int mode = entry.getUnixMode(); + int type = mode & UnixStat.FILE_TYPE_FLAG; + if (entry.isUnixSymlink() || (type != 0 && type != UnixStat.FILE_FLAG && type != UnixStat.DIR_FLAG)) { + throw entryDenied("ZIP links, devices, and other special entries are not allowed."); + } + } + + private void validateTarType(TarArchiveEntry entry) { + if (!entry.isCheckSumOK()) { + throw invalidArchive("TAR entry checksum is invalid.", null); + } + if (entry.isSymbolicLink() || entry.isLink() || entry.isBlockDevice() || entry.isCharacterDevice() + || entry.isFIFO() || entry.isSparse() || (!entry.isFile() && !entry.isDirectory())) { + throw entryDenied("TAR links, devices, FIFO, sparse, and special entries are not allowed."); + } + } + + private String strictZipEntryName(ZipArchiveEntry entry) { + byte[] raw = entry.getRawName(); + if (raw == null) { + throw invalidArchive("ZIP entry name bytes are missing.", null); + } + try { + String decoded = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(raw)).toString(); + if (!decoded.equals(entry.getName())) { + throw invalidArchive("ZIP entry name is not unambiguous UTF-8.", null); + } + return normalizeEntryName(decoded); + } catch (CharacterCodingException error) { + throw invalidArchive("ZIP entry name is not valid UTF-8.", error); + } + } + + private String normalizeEntryName(String rawName) { + if (rawName == null || rawName.isBlank() || rawName.indexOf('\0') >= 0) { + throw entryDenied("Archive entry path is empty or contains NUL."); + } + if (rawName.indexOf('\\') >= 0) { + throw entryDenied("Archive entry path must use forward slashes."); + } + String value = rawName; + while (value.startsWith("./")) { + value = value.substring(2); + } + while (value.endsWith("/")) { + value = value.substring(0, value.length() - 1); + } + if (value.isBlank() || value.startsWith("/") || value.startsWith("~") + || WINDOWS_ABSOLUTE.matcher(value).matches()) { + throw entryDenied("Archive entry path must be workspace-relative."); + } + String[] segments = value.split("/", -1); + if (segments.length > MAX_PATH_DEPTH || value.length() > MAX_PATH_LENGTH) { + throw entryDenied("Archive entry path exceeds the safety limit."); + } + for (String segment : segments) { + if (segment.isEmpty() || ".".equals(segment) || "..".equals(segment)) { + throw entryDenied("Archive entry path contains an unsafe segment."); + } + } + return String.join("/", segments); + } + + private void enforceZipCentralDirectoryCount(Path archive) throws IOException { + long size = Files.size(archive); + int tailSize = (int) Math.min(size, 65_557L); + byte[] tail = new byte[tailSize]; + try (FileChannel channel = FileChannel.open(archive, StandardOpenOption.READ)) { + channel.position(size - tailSize); + ByteBuffer buffer = ByteBuffer.wrap(tail); + while (buffer.hasRemaining() && channel.read(buffer) >= 0) { + // 读取 ZIP 末尾固定有界窗口。 + } + } + for (int index = tail.length - 22; index >= 0; index--) { + if (littleEndianInt(tail, index) != 0x06054b50) { + continue; + } + int commentLength = littleEndianShort(tail, index + 20); + if (index + 22 + commentLength != tail.length) { + continue; + } + int entries = littleEndianShort(tail, index + 10); + if (entries == 0xffff || entries > quotaGuard.maxArchiveEntries()) { + throw limitExceeded("ZIP contains too many entries."); + } + return; + } + throw invalidArchive("ZIP end-of-central-directory record is missing.", null); + } + + private int littleEndianInt(byte[] value, int offset) { + if (offset < 0 || offset + 4 > value.length) { + return -1; + } + return (value[offset] & 0xff) | ((value[offset + 1] & 0xff) << 8) + | ((value[offset + 2] & 0xff) << 16) | ((value[offset + 3] & 0xff) << 24); + } + + private int littleEndianShort(byte[] value, int offset) { + return (value[offset] & 0xff) | ((value[offset + 1] & 0xff) << 8); + } + + private ParsedFileList parseFileList(List arguments, Set allowedOptions) { + Set options = new LinkedHashSet<>(); + List files = new ArrayList<>(); + boolean endOfOptions = false; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if (!endOfOptions && "--".equals(argument)) { + endOfOptions = true; + } else if (!endOfOptions && argument.startsWith("-")) { + if (!allowedOptions.contains(argument) || !options.add(argument)) { + throw denied("Archive command option is not allowed."); + } + } else { + files.add(argument); + } + } + return new ParsedFileList(options, files); + } + + private ParsedZip parseZip(List arguments) { + boolean quiet = false; + boolean recursive = false; + List operands = new ArrayList<>(); + boolean endOfOptions = false; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if (!endOfOptions && "--".equals(argument)) { + endOfOptions = true; + } else if (!endOfOptions && argument.startsWith("-")) { + for (int flag = 1; flag < argument.length(); flag++) { + if (argument.charAt(flag) == 'q') { + quiet = true; + } else if (argument.charAt(flag) == 'r') { + recursive = true; + } else { + throw denied("zip option is not allowed."); + } + } + } else { + operands.add(argument); + } + } + if (operands.size() < 2) { + throw denied("zip requires an archive path and at least one input."); + } + return new ParsedZip(quiet, recursive, operands.get(0), operands.subList(1, operands.size())); + } + + private ParsedExtract parseUnzip(List arguments) { + boolean quiet = false; + String archive = null; + String destination = "."; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if ("-q".equals(argument) && archive == null && !quiet) { + quiet = true; + } else if (archive == null && !argument.startsWith("-")) { + archive = argument; + } else if ("-d".equals(argument) && index + 1 < arguments.size() && ".".equals(destination)) { + destination = arguments.get(++index); + } else { + throw denied("unzip option or operand is not allowed."); + } + } + if (archive == null) { + throw denied("unzip requires one archive path."); + } + return new ParsedExtract(quiet, archive, destination); + } + + private ParsedTar parseTar(List arguments) { + if (arguments.size() < 3) { + throw denied("tar requires a fixed mode and archive path."); + } + TarMode mode = TarMode.parse(arguments.get(1)); + String archive = arguments.get(2); + String destination = null; + List inputs = new ArrayList<>(); + for (int index = 3; index < arguments.size(); index++) { + String argument = arguments.get(index); + if ("-C".equals(argument) && destination == null && index + 1 < arguments.size()) { + destination = arguments.get(++index); + } else if (argument.startsWith("-")) { + throw denied("tar option is not allowed."); + } else { + inputs.add(argument); + } + } + if (mode.create() && inputs.isEmpty()) { + throw denied("tar creation requires at least one input."); + } + if (!mode.create() && !inputs.isEmpty()) { + throw denied("tar extraction and listing do not accept member filters."); + } + if (mode.list() && destination != null) { + throw denied("tar listing does not accept -C."); + } + return new ParsedTar(mode, archive, destination, inputs); + } + + private Path newOutputTarget(String value) { + Path target = pathGuard.resolveForWrite(value); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("ARCHIVE_TARGET_CONFLICT", + "Archive output target already exists.", false); + } + return target; + } + + private Path resolveDestination(String value) { + Path destination = pathGuard.resolveCommandPath(value == null ? "." : value); + if (Files.exists(destination, LinkOption.NOFOLLOW_LINKS) + && !Files.isDirectory(destination, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("ARCHIVE_TARGET_CONFLICT", + "Archive destination is not a directory.", false); + } + return destination; + } + + private String gunzipTarget(String value) { + if (value == null || !value.toLowerCase(Locale.ROOT).endsWith(".gz") || value.length() <= 3) { + throw denied("gunzip input must end with .gz."); + } + return value.substring(0, value.length() - 3); + } + + private long transfer(InputStream input, + OutputStream output, + long limit, + long deadlineNanos, + CRC32 crc) throws IOException { + byte[] buffer = new byte[COPY_BUFFER_SIZE]; + long total = 0; + int read; + while ((read = input.read(buffer)) >= 0) { + checkDeadline(deadlineNanos); + if (read == 0) { + continue; + } + if (total > limit - read) { + throw limitExceeded("Archive content exceeds the configured size limit."); + } + output.write(buffer, 0, read); + if (crc != null) { + crc.update(buffer, 0, read); + } + total += read; + } + return total; + } + + private OutputStream limitedFileOutput(Path path, long limit) throws IOException { + return new LimitedOutputStream(Files.newOutputStream( + path, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING), limit); + } + + private void ensureEntryCount(int count) { + if (count > quotaGuard.maxArchiveEntries()) { + throw limitExceeded("Archive contains too many entries."); + } + } + + private int totalEntryCounter(List plans) { + return plans.size() + 1; + } + + private void ensureSingleSize(long size) { + if (size < 0 || size > quotaGuard.maxArchiveSingleFileSize()) { + throw limitExceeded("Archive entry exceeds the single-file limit."); + } + } + + private long addExpanded(long current, long size) { + ensureSingleSize(size); + long total; + try { + total = Math.addExact(current, size); + } catch (ArithmeticException error) { + throw limitExceeded("Archive expanded size overflowed."); + } + if (total > quotaGuard.maxArchiveTotalSize()) { + throw limitExceeded("Archive exceeds the expanded total-size limit."); + } + return total; + } + + private void checkDeadline(long deadlineNanos) { + if (Thread.currentThread().isInterrupted() || System.nanoTime() > deadlineNanos) { + throw new WorkspaceToolException("SHELL_TIMEOUT", + "Archive command exceeded its timeout.", true); + } + } + + private void forceFile(Path path) throws IOException { + try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE)) { + channel.force(true); + } + } + + private void forceDirectory(Path directory) { + if (directory == null) { + return; + } + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } catch (IOException | UnsupportedOperationException ignored) { + // 某些文件系统不支持目录 fsync;文件已完成 force 和原子 rename。 + } + } + + private void atomicMove(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException error) { + throw new WorkspaceToolException("ARCHIVE_ATOMIC_MOVE_UNSUPPORTED", + "Workspace filesystem does not support atomic archive commits.", false, error); + } + } + + private ArchiveExecutionResult textResult(String value) { + OutputCollector output = new OutputCollector(maxOutputBytes); + output.append(value); + return output.result(); + } + + private WorkspaceToolException denied(String message) { + return new WorkspaceToolException("SHELL_COMMAND_DENIED", message, false); + } + + private WorkspaceToolException entryDenied(String message) { + return new WorkspaceToolException("ARCHIVE_ENTRY_DENIED", message, false); + } + + private WorkspaceToolException limitExceeded(String message) { + return new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", message, false); + } + + private WorkspaceToolException invalidArchive(String message, Throwable cause) { + return cause == null + ? new WorkspaceToolException("ARCHIVE_INVALID", message, false) + : new WorkspaceToolException("ARCHIVE_INVALID", message, false, cause); + } + + /** + * 有界归档命令结果。 + * + * @param output 文本输出 + * @param truncated 是否截断 + */ + record ArchiveExecutionResult(String output, boolean truncated) { + } + + /** + * 归档提交观察器。 + */ + @FunctionalInterface + interface ArchiveCommitObserver { + + /** + * 在一个输出完成原子移动后回调。 + * + * @param committedCount 已提交输出数 + */ + void afterOutputCommitted(int committedCount); + + /** + * 获取无操作观察器。 + * + * @return 无操作观察器 + */ + static ArchiveCommitObserver noop() { + return committedCount -> { + }; + } + } + + private record ParsedFileList(Set options, List files) { + } + + private record ParsedZip(boolean quiet, boolean recursive, String archive, List inputs) { + } + + private record ParsedExtract(boolean quiet, String archive, String destination) { + } + + private record ParsedTar(TarMode mode, String archive, String destination, List inputs) { + } + + private record ArchiveSource(String name, Path snapshot, boolean directory, long size) { + } + + private record EntryPlan(String name, Path target, boolean directory, long size, long crc) { + } + + private record OutputPlan(Path target, Path staged) { + } + + private record DeletionPlan(Path target, Fingerprint fingerprint) { + } + + private record Snapshot(Path path, long size, Fingerprint fingerprint) { + } + + private record Fingerprint(Object fileKey, long size, long modifiedMillis) { + + private static Fingerprint from(BasicFileAttributes attributes) { + return new Fingerprint(attributes.fileKey(), attributes.size(), + attributes.lastModifiedTime().toMillis()); + } + + private boolean matches(BasicFileAttributes attributes) { + return attributes.isRegularFile() && size == attributes.size() + && modifiedMillis == attributes.lastModifiedTime().toMillis() + && (fileKey == null || fileKey.equals(attributes.fileKey())); + } + } + + private enum TarMode { + CREATE(false, true, false), + CREATE_GZIP(true, true, false), + EXTRACT(false, false, false), + EXTRACT_GZIP(true, false, false), + LIST(false, false, true), + LIST_GZIP(true, false, true); + + private final boolean gzip; + private final boolean create; + private final boolean list; + + TarMode(boolean gzip, boolean create, boolean list) { + this.gzip = gzip; + this.create = create; + this.list = list; + } + + private static TarMode parse(String value) { + return switch (value) { + case "-cf" -> CREATE; + case "-czf" -> CREATE_GZIP; + case "-xf" -> EXTRACT; + case "-xzf" -> EXTRACT_GZIP; + case "-tf" -> LIST; + case "-tzf" -> LIST_GZIP; + default -> throw new WorkspaceToolException("SHELL_COMMAND_DENIED", + "tar mode is not allowed.", false); + }; + } + + private boolean gzip() { + return gzip; + } + + private boolean create() { + return create; + } + + private boolean list() { + return list; + } + } + + private static final class EntryIndex { + + private final Set exact = new HashSet<>(); + private final Set files = new HashSet<>(); + private final Map firstDescendant = new HashMap<>(); + + private void add(String rawName, boolean directory) { + String name = rawName; + if (!exact.add(name)) { + throw new WorkspaceToolException("ARCHIVE_DUPLICATE_ENTRY", + "Archive contains a duplicate entry.", false); + } + String ancestor = name; + int separator = ancestor.indexOf('/'); + while (separator >= 0) { + String prefix = ancestor.substring(0, separator); + if (files.contains(prefix)) { + throw new WorkspaceToolException("ARCHIVE_ENTRY_DENIED", + "Archive contains a file/directory hierarchy conflict.", false); + } + firstDescendant.putIfAbsent(prefix, name); + separator = ancestor.indexOf('/', separator + 1); + } + if (!directory && firstDescendant.containsKey(name)) { + throw new WorkspaceToolException("ARCHIVE_ENTRY_DENIED", + "Archive contains a file/directory hierarchy conflict.", false); + } + if (!directory) { + files.add(name); + } + } + } + + private static final class LimitedOutputStream extends FilterOutputStream { + + private final long limit; + private long count; + + private LimitedOutputStream(OutputStream output, long limit) { + super(output); + this.limit = limit; + } + + @Override + public void write(int value) throws IOException { + ensureCapacity(1); + out.write(value); + count++; + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + ensureCapacity(length); + out.write(value, offset, length); + count += length; + } + + private void ensureCapacity(int additional) { + if (additional < 0 || count > limit - additional) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Archive output exceeds the single-file limit.", false); + } + } + } + + private static final class OutputCollector { + + private final int limit; + private final StringBuilder value = new StringBuilder(); + private int bytes; + private boolean truncated; + + private OutputCollector(int limit) { + this.limit = limit; + } + + private OutputCollector append(char character) { + return append(String.valueOf(character)); + } + + private OutputCollector append(String text) { + if (truncated || text == null || text.isEmpty()) { + return this; + } + for (int index = 0; index < text.length(); ) { + int codePoint = text.codePointAt(index); + String current = new String(Character.toChars(codePoint)); + int encoded = current.getBytes(StandardCharsets.UTF_8).length; + if (bytes > limit - encoded) { + truncated = true; + break; + } + value.append(current); + bytes += encoded; + index += Character.charCount(codePoint); + } + return this; + } + + private ArchiveExecutionResult result() { + return new ArchiveExecutionResult(value.toString(), truncated); + } + } + + private static final class Stage implements AutoCloseable { + + private final Path root; + + private Stage(Path root) { + this.root = root; + } + + private static Stage create(Path workspaceRoot) throws IOException { + Path parent = workspaceRoot.getParent(); + if (parent == null) { + throw new WorkspaceToolException("WORKSPACE_CONFIG_INVALID", + "Workspace root must have a parent directory for archive staging.", false); + } + return new Stage(Files.createTempDirectory(parent, ".easyagents-archive-stage-")); + } + + private Path newFile(String prefix) throws IOException { + return Files.createTempFile(root, prefix + UUID.randomUUID(), ".tmp"); + } + + @Override + public void close() { + try (Stream paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } catch (IOException error) { + // 调用结果已确定;完整堆栈仅记录到服务端,不向模型泄露宿主路径。 + logger.error("Failed to clean safe archive staging directory", error); + } + } + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeReadFileTool.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeReadFileTool.java new file mode 100644 index 0000000..edb58d7 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeReadFileTool.java @@ -0,0 +1,290 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.AgentTool; +import io.agentscope.core.tool.ToolCallParam; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.stream.Stream; + +/** + * 与 AgentScope 1.x 文件读取 Schema 兼容的工作区安全工具。 + */ +final class SafeReadFileTool { + + private final ViewTextFileTool viewTextFileTool; + private final ListDirectoryTool listDirectoryTool; + + /** + * 创建文件读取工具组。 + * + * @param pathGuard 路径保护器 + * @param quotaGuard 配额保护器 + */ + SafeReadFileTool(WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + this.viewTextFileTool = new ViewTextFileTool(pathGuard, quotaGuard); + this.listDirectoryTool = new ListDirectoryTool(pathGuard, quotaGuard); + } + + /** + * 获取查看文本文件工具。 + * + * @return AgentScope 工具 + */ + AgentTool viewTextFileTool() { + return viewTextFileTool; + } + + /** + * 获取列目录工具。 + * + * @return AgentScope 工具 + */ + AgentTool listDirectoryTool() { + return listDirectoryTool; + } + + /** + * 查看工作区文本文件。 + */ + private static final class ViewTextFileTool implements AgentTool { + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + + private ViewTextFileTool(WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + } + + /** + * 获取工具名。 + * + * @return `view_text_file` + */ + @Override + public String getName() { + return AgentOperateToolAdapter.VIEW_TEXT_FILE_TOOL; + } + + /** + * 获取工具描述。 + * + * @return 工具描述 + */ + @Override + public String getDescription() { + return "View UTF-8 text file content in the workspace with optional line ranges."; + } + + /** + * 获取与 AgentScope 1.x 兼容的参数 Schema。 + * + * @return JSON Schema + */ + @Override + public Map getParameters() { + Map properties = new LinkedHashMap<>(); + properties.put("file_path", Map.of("type", "string", "description", "The target file path")); + properties.put("ranges", Map.of( + "type", "string", + "description", "Optional inclusive line range such as '1,100' or '-100,-1'")); + return Map.of("type", "object", "properties", properties, "required", List.of("file_path")); + } + + /** + * 读取并格式化指定行范围。 + * + * @param param Tool 调用参数 + * @return Tool 结果 + */ + @Override + public Mono callAsync(ToolCallParam param) { + return Mono.fromCallable(() -> view(param)).subscribeOn(Schedulers.boundedElastic()); + } + + private ToolResultBlock view(ToolCallParam param) { + try { + String filePath = requiredString(param, "file_path"); + String ranges = optionalString(param, "ranges"); + Path target = pathGuard.resolveExistingFile(filePath); + WorkspaceTextFiles.RangedLines rangedLines = WorkspaceTextFiles.readUtf8Lines( + target, ranges, quotaGuard.maxReadSize()); + quotaGuard.validateRangeRead(target, rangedLines.readBytes()); + StringBuilder content = new StringBuilder(); + for (int index = 0; index < rangedLines.lines().size(); index++) { + content.append(rangedLines.startLine() + index).append(": ") + .append(rangedLines.lines().get(index)).append('\n'); + } + int endLine = rangedLines.lines().isEmpty() + ? rangedLines.startLine() - 1 + : rangedLines.startLine() + rangedLines.lines().size() - 1; + return ToolResultBlock.text("The content of " + pathGuard.display(target) + + " in lines [" + rangedLines.startLine() + ", " + endLine + "]:\n```\n" + + content + "```"); + } catch (AgentRuntimeException error) { + return WorkspaceToolResults.error(error); + } catch (RuntimeException error) { + return WorkspaceToolResults.error( + new AgentRuntimeException("Unexpected workspace read failure.", error)); + } + } + } + + /** + * 列出工作区单层目录内容。 + */ + private static final class ListDirectoryTool implements AgentTool { + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + + private ListDirectoryTool(WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + } + + /** + * 获取工具名。 + * + * @return `list_directory` + */ + @Override + public String getName() { + return AgentOperateToolAdapter.LIST_DIRECTORY_TOOL; + } + + /** + * 获取工具描述。 + * + * @return 工具描述 + */ + @Override + public String getDescription() { + return "List one level of files and directories using workspace-relative paths."; + } + + /** + * 获取与 AgentScope 1.x 兼容的参数 Schema。 + * + * @return JSON Schema + */ + @Override + public Map getParameters() { + return Map.of( + "type", "object", + "properties", Map.of("dir_path", Map.of( + "type", "string", "description", "The target directory path")), + "required", List.of("dir_path")); + } + + /** + * 列出单层目录。 + * + * @param param Tool 调用参数 + * @return Tool 结果 + */ + @Override + public Mono callAsync(ToolCallParam param) { + return Mono.fromCallable(() -> list(param)).subscribeOn(Schedulers.boundedElastic()); + } + + private ToolResultBlock list(ToolCallParam param) { + try { + Path directory = pathGuard.resolveExistingDirectory(requiredString(param, "dir_path")); + quotaGuard.validateCurrentUsage(); + int limit = quotaGuard.maxDirectoryEntries(); + Comparator displayOrder = Comparator.comparing(pathGuard::display); + PriorityQueue retained = new PriorityQueue<>(limit, displayOrder.reversed()); + long entryCount = 0; + try (Stream stream = Files.list(directory)) { + for (Path entry : (Iterable) stream::iterator) { + entryCount++; + if (retained.size() < limit) { + retained.add(entry); + } else if (displayOrder.compare(entry, retained.peek()) < 0) { + retained.poll(); + retained.add(entry); + } + } + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace directory cannot be listed.", true, error); + } + List entries = new ArrayList<>(retained); + entries.sort(displayOrder); + StringBuilder result = new StringBuilder("Contents of directory ") + .append(pathGuard.display(directory)).append(":\n"); + boolean truncated = entryCount > limit; + for (Path entry : entries) { + String type; + long size = 0; + if (Files.isSymbolicLink(entry)) { + type = "blocked-symlink"; + } else if (Files.isDirectory(entry, LinkOption.NOFOLLOW_LINKS)) { + type = "directory"; + } else if (Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) { + type = "file"; + try { + size = Files.size(entry); + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace entry size cannot be inspected.", true, error); + } + } else { + type = "blocked-non-regular"; + } + result.append(type).append('\t').append(pathGuard.display(entry)); + if ("file".equals(type)) { + result.append('\t').append(size).append(" bytes"); + } + result.append('\n'); + } + if (truncated) { + result.append("Truncated: true; limit=") + .append(limit).append('\n'); + } + return ToolResultBlock.text(result.toString()); + } catch (AgentRuntimeException error) { + return WorkspaceToolResults.error(error); + } catch (RuntimeException error) { + return WorkspaceToolResults.error( + new AgentRuntimeException("Unexpected workspace listing failure.", error)); + } + } + } + + private static String requiredString(ToolCallParam param, String name) { + Object value = param == null ? null : param.getInput().get(name); + if (!(value instanceof String text) || text.isBlank()) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Missing required string parameter: " + name, false); + } + return text; + } + + private static String optionalString(ToolCallParam param, String name) { + Object value = param == null ? null : param.getInput().get(name); + if (value == null) { + return null; + } + if (!(value instanceof String text)) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Invalid string parameter: " + name, false); + } + return text; + } + +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeWriteFileTool.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeWriteFileTool.java new file mode 100644 index 0000000..4bdea07 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/SafeWriteFileTool.java @@ -0,0 +1,325 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.AgentTool; +import io.agentscope.core.tool.ToolCallParam; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 与 AgentScope 1.x 文件写入 Schema 兼容的原子工作区工具。 + */ +final class SafeWriteFileTool { + + private final WriteTextFileTool writeTextFileTool; + private final InsertTextFileTool insertTextFileTool; + + /** + * 创建文件写入工具组。 + * + * @param pathGuard 路径保护器 + * @param quotaGuard 配额保护器 + */ + SafeWriteFileTool(WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + this.writeTextFileTool = new WriteTextFileTool(pathGuard, quotaGuard); + this.insertTextFileTool = new InsertTextFileTool(pathGuard, quotaGuard); + } + + /** + * 获取写入文本文件工具。 + * + * @return AgentScope 工具 + */ + AgentTool writeTextFileTool() { + return writeTextFileTool; + } + + /** + * 获取插入文本文件工具。 + * + * @return AgentScope 工具 + */ + AgentTool insertTextFileTool() { + return insertTextFileTool; + } + + /** + * 新建、覆盖或范围替换文本文件。 + */ + private static final class WriteTextFileTool implements AgentTool { + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + + private WriteTextFileTool(WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + } + + /** + * 获取工具名。 + * + * @return `write_text_file` + */ + @Override + public String getName() { + return AgentOperateToolAdapter.WRITE_TEXT_FILE_TOOL; + } + + /** + * 获取工具描述。 + * + * @return 工具描述 + */ + @Override + public String getDescription() { + return "Create, overwrite, or replace an inclusive line range in a UTF-8 workspace file."; + } + + /** + * 获取与 AgentScope 1.x 兼容的参数 Schema。 + * + * @return JSON Schema + */ + @Override + public Map getParameters() { + Map properties = new LinkedHashMap<>(); + properties.put("file_path", Map.of("type", "string", "description", "The target file path")); + properties.put("content", Map.of("type", "string", "description", "The content to be written")); + properties.put("ranges", Map.of( + "type", "string", + "description", "Optional inclusive replacement range such as '1,5'")); + return Map.of( + "type", "object", + "properties", properties, + "required", List.of("file_path", "content")); + } + + /** + * 原子写入文件。 + * + * @param param Tool 调用参数 + * @return Tool 结果 + */ + @Override + public Mono callAsync(ToolCallParam param) { + return Mono.fromCallable(() -> write(param)).subscribeOn(Schedulers.boundedElastic()); + } + + private ToolResultBlock write(ToolCallParam param) { + try { + String filePath = requiredString(param, "file_path"); + String content = requiredStringAllowEmpty(param, "content"); + String ranges = optionalString(param, "ranges"); + Path target = pathGuard.resolveForWrite(filePath); + byte[] bytes; + if (ranges == null || ranges.isBlank() || !Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + bytes = content.getBytes(StandardCharsets.UTF_8); + } else { + quotaGuard.validateFullRead(target); + List lines = splitLines(WorkspaceTextFiles.readUtf8(target)); + int[] range = parseReplacementRange(ranges, lines.size()); + List updated = new ArrayList<>(); + updated.addAll(lines.subList(0, range[0] - 1)); + updated.addAll(splitContentLines(content)); + updated.addAll(lines.subList(range[1], lines.size())); + bytes = String.join("\n", updated).getBytes(StandardCharsets.UTF_8); + } + quotaGuard.validateWrite(target, bytes.length); + WorkspaceTextFiles.atomicWrite(pathGuard, target, bytes); + return ToolResultBlock.text("Write " + pathGuard.display(target) + " successfully."); + } catch (AgentRuntimeException error) { + return WorkspaceToolResults.error(error); + } catch (RuntimeException error) { + return WorkspaceToolResults.error( + new AgentRuntimeException("Unexpected workspace write failure.", error)); + } + } + } + + /** + * 在指定 1-based 行号插入文本。 + */ + private static final class InsertTextFileTool implements AgentTool { + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaGuard quotaGuard; + + private InsertTextFileTool(WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + this.pathGuard = pathGuard; + this.quotaGuard = quotaGuard; + } + + /** + * 获取工具名。 + * + * @return `insert_text_file` + */ + @Override + public String getName() { + return AgentOperateToolAdapter.INSERT_TEXT_FILE_TOOL; + } + + /** + * 获取工具描述。 + * + * @return 工具描述 + */ + @Override + public String getDescription() { + return "Insert UTF-8 content at a 1-based line number in an existing workspace file."; + } + + /** + * 获取与 AgentScope 1.x 兼容的参数 Schema。 + * + * @return JSON Schema + */ + @Override + public Map getParameters() { + Map properties = new LinkedHashMap<>(); + properties.put("file_path", Map.of("type", "string", "description", "The target file path")); + properties.put("content", Map.of("type", "string", "description", "The content to be inserted")); + properties.put("line_number", Map.of( + "type", "integer", + "description", "The 1-based line number where content is inserted")); + return Map.of( + "type", "object", + "properties", properties, + "required", List.of("file_path", "content", "line_number")); + } + + /** + * 原子插入文件内容。 + * + * @param param Tool 调用参数 + * @return Tool 结果 + */ + @Override + public Mono callAsync(ToolCallParam param) { + return Mono.fromCallable(() -> insert(param)).subscribeOn(Schedulers.boundedElastic()); + } + + private ToolResultBlock insert(ToolCallParam param) { + try { + String filePath = requiredString(param, "file_path"); + String content = requiredStringAllowEmpty(param, "content"); + int lineNumber = requiredInteger(param, "line_number"); + Path target = pathGuard.resolveExistingFile(filePath); + quotaGuard.validateFullRead(target); + List lines = splitLines(WorkspaceTextFiles.readUtf8(target)); + if (lineNumber < 1 || lineNumber > lines.size() + 1) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "line_number is outside the valid range [1, " + + (lines.size() + 1) + "].", false); + } + List updated = new ArrayList<>(lines); + updated.addAll(lineNumber - 1, splitContentLines(content)); + byte[] bytes = String.join("\n", updated).getBytes(StandardCharsets.UTF_8); + quotaGuard.validateWrite(target, bytes.length); + WorkspaceTextFiles.atomicWrite(pathGuard, target, bytes); + return ToolResultBlock.text("Insert content into " + pathGuard.display(target) + + " at line " + lineNumber + " successfully."); + } catch (AgentRuntimeException error) { + return WorkspaceToolResults.error(error); + } catch (RuntimeException error) { + return WorkspaceToolResults.error( + new AgentRuntimeException("Unexpected workspace insert failure.", error)); + } + } + } + + private static String requiredString(ToolCallParam param, String name) { + String text = requiredStringAllowEmpty(param, name); + if (text.isBlank()) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Missing required string parameter: " + name, false); + } + return text; + } + + private static String requiredStringAllowEmpty(ToolCallParam param, String name) { + Object value = param == null ? null : param.getInput().get(name); + if (!(value instanceof String text)) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Missing required string parameter: " + name, false); + } + return text; + } + + private static String optionalString(ToolCallParam param, String name) { + Object value = param == null ? null : param.getInput().get(name); + if (value == null) { + return null; + } + if (!(value instanceof String text)) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Invalid string parameter: " + name, false); + } + return text; + } + + private static int requiredInteger(ToolCallParam param, String name) { + Object value = param == null ? null : param.getInput().get(name); + if (!(value instanceof Number number)) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Missing required integer parameter: " + name, false); + } + return number.intValue(); + } + + private static int[] parseReplacementRange(String ranges, int lineCount) { + String normalized = ranges.trim().replace("[", "").replace("]", ""); + String[] parts = normalized.split(",", -1); + if (parts.length != 2) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Invalid range format. Expected 'start,end'.", false); + } + try { + int start = Integer.parseInt(parts[0].trim()); + int end = Integer.parseInt(parts[1].trim()); + if (start < 1 || end < start || start > lineCount || end > lineCount) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Replacement range is outside the file.", false); + } + return new int[]{start, end}; + } catch (NumberFormatException error) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Invalid range format. Expected integer line numbers.", false, error); + } + } + + private static List splitLines(String content) { + if (content.isEmpty()) { + return new ArrayList<>(); + } + String normalized = content.replace("\r\n", "\n").replace('\r', '\n'); + String[] values = normalized.split("\n", -1); + int length = values.length; + if (length > 0 && values[length - 1].isEmpty()) { + length--; + } + List lines = new ArrayList<>(length); + for (int index = 0; index < length; index++) { + lines.add(values[index]); + } + return lines; + } + + private static List splitContentLines(String content) { + if (content.isEmpty()) { + return List.of(""); + } + return List.of(content.replace("\r\n", "\n").replace('\r', '\n').split("\n", -1)); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellCommandOptionValidator.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellCommandOptionValidator.java new file mode 100644 index 0000000..3d39dcd --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellCommandOptionValidator.java @@ -0,0 +1,601 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * 白名单命令的命令级选项与路径参数校验器。 + * + *

入口命令白名单不足以阻止工具通过合法命令的扩展选项启动子进程或访问第二路径。 + * 该校验器集中关闭这些二级执行入口,并对已知文件参数执行工作区路径保护。 + */ +final class ShellCommandOptionValidator { + + private static final Pattern AWK_CODE_EXECUTION = Pattern.compile( + "(?is).*(\\bsystem\\s*\\(|\\bgetline\\b|\\bENVIRON\\b|@load\\b|\\bextension\\s*\\().*"); + private static final Pattern SED_SIDE_EFFECT_COMMAND = Pattern.compile( + "(?is).*(^|[;{}\\n])\\s*(?:(?:\\d+|\\$|/([^/\\n\\\\]|\\\\.)*/)(?:\\s*,\\s*" + + "(?:\\d+|\\$|/([^/\\n\\\\]|\\\\.)*/))?\\s*)?[eErRwW](?:\\s|$).*"); + private static final Pattern JQ_EXTERNAL_INPUT = Pattern.compile( + "(?is).*(\\b(import|include|module|input|inputs|env)\\b|\\$ENV\\b).*"); + + private final WorkspacePathGuard pathGuard; + + /** + * 创建命令选项校验器。 + * + * @param pathGuard 工作区路径保护器 + */ + ShellCommandOptionValidator(WorkspacePathGuard pathGuard) { + this.pathGuard = pathGuard; + } + + /** + * 校验命令专属的子执行入口、文件选项和路径操作数。 + * + * @param arguments 已完成安全分词的命令参数 + */ + void validate(List arguments) { + String command = arguments.get(0); + switch (command) { + case "ls" -> validateList(arguments); + case "awk" -> validateAwk(arguments); + case "sed" -> validateSed(arguments); + case "rg" -> validateRipgrep(arguments); + case "grep" -> validateGrep(arguments); + case "jq" -> validateJq(arguments); + case "sort" -> validateSort(arguments); + case "uniq" -> validateUniq(arguments); + case "diff", "cmp" -> validateExistingOperands(arguments); + case "du" -> validateDiskUsage(arguments); + case "tree" -> validateTree(arguments); + case "cp" -> validateCopy(arguments); + case "mkdir", "touch", "mv", "rm" -> validateAllOperands(arguments); + case "wc" -> validateWordCount(arguments); + case "file" -> validateFile(arguments); + case "sha256sum", "shasum" -> validateChecksum(arguments); + case "tail" -> validateTail(arguments); + case "pandoc" -> validatePandoc(arguments); + case "soffice" -> validateSoffice(arguments); + case "pdftoppm" -> validatePdfToPpm(arguments); + case "pdfinfo" -> validatePdfInfo(arguments); + case "pdftotext" -> validatePdfToText(arguments); + case "pdfimages" -> validatePdfImages(arguments); + case "qpdf" -> validateQpdf(arguments); + case "cat", "head", "cut", "stat" -> + validateExistingOperands(arguments); + default -> { + // pwd/date/tr/basename/dirname/python/python3/node 没有额外的子执行选项;脚本入口由外层单独校验。 + } + } + } + + private void validateDiskUsage(List arguments) { + rejectOptions(arguments, Set.of( + "--files0-from", "--exclude-from", "-L", "--dereference", "-H", "-D", + "--dereference-args")); + validateExistingOperands(arguments); + } + + private void validateTree(List arguments) { + rejectOptions(arguments, Set.of( + "-l", "--follow-links", "-o", "--fromfile", "--gitfile", "--info")); + validateExistingOperands(arguments); + } + + private void validatePandoc(List arguments) { + rejectOptions(arguments, Set.of( + "-F", "--filter", "-L", "--lua-filter", "-d", "--defaults", "--data-dir", + "--resource-path", "--extract-media", "--pdf-engine", "--pdf-engine-opt")); + for (String argument : arguments.subList(1, arguments.size())) { + if (isAttachedShortOption(argument, "-o")) { + throw new AgentRuntimeException( + "pandoc attached output paths are not allowed; use -o followed by a workspace path."); + } + } + validateFollowingFileOptions(arguments, Set.of( + "--template", "--metadata-file", "--reference-doc", "--syntax-definition", + "--include-in-header", "--include-before-body", "--include-after-body", + "--bibliography", "--csl", "--citation-abbreviations"), false); + validateFollowingFileOptions(arguments, Set.of("-o", "--output", "--log"), true); + validateExistingOperands(arguments); + } + + private void validateSoffice(List arguments) { + String format = null; + String outputDirectory = null; + List inputs = new ArrayList<>(); + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + String option = optionName(argument); + if (Set.of("--headless", "--nologo", "--nodefault", "--nolockcheck", "--norestore") + .contains(option)) { + continue; + } + if ("--convert-to".equals(option)) { + format = optionValue(arguments, index); + if (!argument.contains("=")) { + index++; + } + continue; + } + if ("--outdir".equals(option)) { + outputDirectory = optionValue(arguments, index); + if (!argument.contains("=")) { + index++; + } + continue; + } + if (argument.startsWith("-")) { + throw new AgentRuntimeException("soffice option is not allowed: " + option); + } + inputs.add(argument); + } + if (format == null || outputDirectory == null || inputs.isEmpty()) { + throw new AgentRuntimeException( + "soffice requires --convert-to, --outdir, and at least one workspace input file."); + } + String normalizedFormat = format.split(":", 2)[0].toLowerCase(java.util.Locale.ROOT); + if (!Set.of("pdf", "docx", "xlsx", "pptx", "odt", "ods", "odp", "html", "txt", "csv") + .contains(normalizedFormat)) { + throw new AgentRuntimeException("soffice output format is not allowed: " + normalizedFormat); + } + pathGuard.resolveExistingDirectory(outputDirectory); + inputs.forEach(pathGuard::resolveExistingFile); + } + + private void validatePdfToPpm(List arguments) { + List operands = pdfOperands(arguments, Set.of( + "-f", "-l", "-r", "-rx", "-ry", "-scale-to", "-scale-to-x", "-scale-to-y", + "-x", "-y", "-W", "-H", "-sz")); + if (operands.size() != 2) { + throw new AgentRuntimeException("pdftoppm requires one PDF input and one output prefix."); + } + pathGuard.resolveExistingFile(operands.get(0)); + pathGuard.resolveCommandPath(operands.get(1)); + } + + private void validatePdfInfo(List arguments) { + rejectOptions(arguments, Set.of("-opw", "-upw")); + List operands = pdfOperands(arguments, Set.of("-f", "-l")); + if (operands.size() != 1) { + throw new AgentRuntimeException("pdfinfo requires exactly one workspace PDF input."); + } + pathGuard.resolveExistingFile(operands.get(0)); + } + + private void validatePdfToText(List arguments) { + rejectOptions(arguments, Set.of("-opw", "-upw")); + List operands = pdfOperands(arguments, Set.of( + "-f", "-l", "-r", "-x", "-y", "-W", "-H", "-enc", "-eol")); + if (operands.size() < 1 || operands.size() > 2) { + throw new AgentRuntimeException("pdftotext requires one PDF input and an optional output file."); + } + pathGuard.resolveExistingFile(operands.get(0)); + if (operands.size() == 2 && !"-".equals(operands.get(1))) { + pathGuard.resolveCommandPath(operands.get(1)); + } + } + + private void validatePdfImages(List arguments) { + rejectOptions(arguments, Set.of("-opw", "-upw")); + List operands = pdfOperands(arguments, Set.of("-f", "-l", "-jpegopt")); + if (operands.size() != 2) { + throw new AgentRuntimeException("pdfimages requires one PDF input and one output prefix."); + } + pathGuard.resolveExistingFile(operands.get(0)); + pathGuard.resolveCommandPath(operands.get(1)); + } + + private void validateQpdf(List arguments) { + rejectOptions(arguments, Set.of( + "--replace-input", "--password-file", "--encryption-file-password", + "--copy-attachments-from", "--overlay", "--underlay", "--json-input", + "--job-json-file")); + for (String argument : arguments.subList(1, arguments.size())) { + if (argument.startsWith("@")) { + throw new AgentRuntimeException("qpdf response files are not allowed."); + } + } + validateExistingOperands(arguments); + } + + private List pdfOperands(List arguments, Set optionsWithValues) { + List result = new ArrayList<>(); + boolean endOfOptions = false; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if (!endOfOptions && "--".equals(argument)) { + endOfOptions = true; + continue; + } + if (!endOfOptions && argument.startsWith("-")) { + String option = optionName(argument); + if (optionsWithValues.contains(option) && !argument.contains("=")) { + if (++index >= arguments.size()) { + throw new AgentRuntimeException("PDF command option requires a value: " + option); + } + } + continue; + } + result.add(argument); + } + return result; + } + + private void validateFollowingFileOptions(List arguments, + Set fileOptions, + boolean writable) { + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + String option = optionName(argument); + if (!fileOptions.contains(option)) { + continue; + } + String path = optionValue(arguments, index); + if (writable) { + pathGuard.resolveCommandPath(path); + } else { + pathGuard.resolveExistingFile(path); + } + if (!argument.contains("=")) { + index++; + } + } + } + + private void validateAwk(List arguments) { + rejectOptions(arguments, Set.of( + "-f", "--file", "-e", "--exec", "-i", "--include", "-l", "--load", "-W", + "-d", "--dump-variables", "-o", "--pretty-print", "-p", "--profile")); + for (String argument : operands(arguments)) { + if (AWK_CODE_EXECUTION.matcher(argument).matches()) { + throw new AgentRuntimeException("awk sub-process and external input features are not allowed."); + } + } + validateExistingOperandsSkippingFirst(arguments); + } + + private void validateSed(List arguments) { + List expressions = new ArrayList<>(); + List files = new ArrayList<>(); + boolean endOfOptions = false; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if (!endOfOptions && "--".equals(argument)) { + endOfOptions = true; + continue; + } + if (!endOfOptions && (argument.equals("-i") || argument.startsWith("-i") + || argument.startsWith("--in-place") || argument.equals("--follow-symlinks") + || argument.startsWith("-f") || argument.startsWith("--file"))) { + throw new AgentRuntimeException("sed in-place, external script, and symlink-following options are not allowed."); + } + if (!endOfOptions && ("-e".equals(argument) || "--expression".equals(argument))) { + if (++index >= arguments.size()) { + throw new AgentRuntimeException("sed expression option requires a value."); + } + expressions.add(arguments.get(index)); + continue; + } + if (!endOfOptions && argument.startsWith("--expression=")) { + expressions.add(argument.substring("--expression=".length())); + continue; + } + if (!endOfOptions && argument.startsWith("-") && !isSafeSedFlag(argument)) { + throw new AgentRuntimeException("sed option is not allowed."); + } + if (!endOfOptions && argument.startsWith("-")) { + continue; + } + if (expressions.isEmpty()) { + expressions.add(argument); + } else { + files.add(argument); + } + } + if (expressions.isEmpty()) { + throw new AgentRuntimeException("sed requires an inline expression."); + } + for (String expression : expressions) { + if (SED_SIDE_EFFECT_COMMAND.matcher(expression).matches() + || containsUnsafeSubstitutionFlag(expression)) { + throw new AgentRuntimeException("sed execute/read/write commands are not allowed."); + } + } + for (String file : files) { + validateExistingPath(file); + } + } + + private void validateRipgrep(List arguments) { + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + String option = optionName(argument); + if (Set.of("--pre", "--pre-glob", "--hostname-bin", "--search-zip").contains(option) + || isShortOptionPresent(argument, 'z') || "--follow".equals(option) + || isShortOptionPresent(argument, 'L')) { + throw new AgentRuntimeException( + "rg preprocessors, archive search, and symlink-following options are not allowed."); + } + if (Set.of("-f", "--file", "--ignore-file").contains(option) + || isAttachedShortOption(argument, "-f")) { + String path = attachedOrFollowingValue(arguments, index, "-f"); + validateExistingPath(path); + if (!argument.contains("=") && !isAttachedShortOption(argument, "-f")) { + index++; + } + } + } + validateExistingOperands(arguments); + } + + private void validateGrep(List arguments) { + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + String option = optionName(argument); + if (isShortOptionPresent(argument, 'R') || "--dereference-recursive".equals(option)) { + throw new AgentRuntimeException("grep symlink-following recursion is not allowed."); + } + if (Set.of("-f", "--file", "--exclude-from").contains(option) + || isAttachedShortOption(argument, "-f")) { + String path = attachedOrFollowingValue(arguments, index, "-f"); + validateExistingPath(path); + if (!argument.contains("=") && !isAttachedShortOption(argument, "-f")) { + index++; + } + } + } + validateExistingOperands(arguments); + } + + private void validateJq(List arguments) { + rejectOptions(arguments, Set.of("-f", "--from-file", "-L", "--library-path", "--run-tests")); + for (String operand : operands(arguments)) { + if (JQ_EXTERNAL_INPUT.matcher(operand).matches()) { + throw new AgentRuntimeException("jq module, environment, and external input functions are not allowed."); + } + } + for (int index = 1; index < arguments.size(); index++) { + String option = optionName(arguments.get(index)); + if (Set.of("--argfile", "--slurpfile", "--rawfile").contains(option)) { + if (index + 2 >= arguments.size()) { + throw new AgentRuntimeException("jq file option requires a variable name and workspace file."); + } + validateExistingPath(arguments.get(index + 2)); + index += 2; + } + } + validateExistingOperandsSkippingFirst(arguments); + } + + private void validateSort(List arguments) { + rejectOptions(arguments, Set.of("-o", "--output", "--compress-program", "-T", "--temporary-directory")); + for (int index = 1; index < arguments.size(); index++) { + String option = optionName(arguments.get(index)); + if ("--random-source".equals(option)) { + String path = optionValue(arguments, index); + validateExistingPath(path); + if (!arguments.get(index).contains("=")) { + index++; + } + } + } + validateExistingOperands(arguments); + } + + private void validateUniq(List arguments) { + List operands = operands(arguments); + if (operands.size() > 1) { + throw new AgentRuntimeException("uniq output-file operand is not allowed; use write_text_file instead."); + } + if (!operands.isEmpty()) { + validateExistingPath(operands.get(0)); + } + } + + private void validateCopy(List arguments) { + for (String argument : arguments) { + if (isShortOptionPresent(argument, 'L') || isShortOptionPresent(argument, 'H') + || isShortOptionPresent(argument, 'l') || isShortOptionPresent(argument, 's') + || Set.of("--dereference", "--link", "--symbolic-link") + .contains(optionName(argument))) { + throw new AgentRuntimeException("cp link creation and symlink-following options are not allowed."); + } + } + validateAllOperands(arguments); + } + + private void validateTail(List arguments) { + for (String argument : arguments.subList(1, arguments.size())) { + String option = optionName(argument); + if (isShortOptionPresent(argument, 'f') || isShortOptionPresent(argument, 'F') + || "--follow".equals(option)) { + throw new AgentRuntimeException("tail follow mode is not allowed."); + } + } + validateExistingOperands(arguments); + } + + private void validateList(List arguments) { + for (String argument : arguments.subList(1, arguments.size())) { + String option = optionName(argument); + if (isShortOptionPresent(argument, 'L') || "--dereference".equals(option) + || "--dereference-command-line".equals(option) + || "--dereference-command-line-symlink-to-dir".equals(option)) { + throw new AgentRuntimeException("ls symlink-following options are not allowed."); + } + } + validateExistingOperands(arguments); + } + + private void validateWordCount(List arguments) { + rejectOptions(arguments, Set.of("--files0-from")); + validateExistingOperands(arguments); + } + + private void validateFile(List arguments) { + rejectOptions(arguments, Set.of("-f", "--files-from", "-C", "--compile")); + validateExistingOperands(arguments); + } + + private void validateChecksum(List arguments) { + rejectOptions(arguments, Set.of("-c", "--check")); + validateExistingOperands(arguments); + } + + private void validateAllOperands(List arguments) { + for (String operand : operands(arguments)) { + pathGuard.resolveCommandPath(operand); + } + } + + private void validateExistingOperands(List arguments) { + for (String operand : operands(arguments)) { + validateExistingPathIfPresent(operand); + } + } + + private void validateExistingOperandsSkippingFirst(List arguments) { + List operands = operands(arguments); + for (int index = 1; index < operands.size(); index++) { + validateExistingPathIfPresent(operands.get(index)); + } + } + + private void validateExistingPathIfPresent(String value) { + java.nio.file.Path candidate = pathGuard.root().resolve(value).normalize(); + if (java.nio.file.Files.exists(candidate, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + pathGuard.resolveExistingEntry(value); + } + } + + private void validateExistingPath(String value) { + pathGuard.resolveExistingFile(value); + } + + private void rejectOptions(List arguments, Set rejected) { + for (String argument : arguments.subList(1, arguments.size())) { + String option = optionName(argument); + if (rejected.contains(option) || rejected.stream() + .filter(value -> value.startsWith("-") && !value.startsWith("--") && value.length() == 2) + .anyMatch(value -> isAttachedShortOption(argument, value))) { + throw new AgentRuntimeException("Command option is not allowed: " + option); + } + } + } + + private List operands(List arguments) { + List operands = new ArrayList<>(); + boolean endOfOptions = false; + for (int index = 1; index < arguments.size(); index++) { + String argument = arguments.get(index); + if (!endOfOptions && "--".equals(argument)) { + endOfOptions = true; + continue; + } + if (!endOfOptions && argument.startsWith("-")) { + continue; + } + operands.add(argument); + } + return operands; + } + + private String optionName(String argument) { + int equals = argument.indexOf('='); + return equals < 0 ? argument : argument.substring(0, equals); + } + + private String optionValue(List arguments, int optionIndex) { + String argument = arguments.get(optionIndex); + int equals = argument.indexOf('='); + if (equals >= 0) { + String value = argument.substring(equals + 1); + if (value.isBlank()) { + throw new AgentRuntimeException("Command file option requires a value."); + } + return value; + } + if (optionIndex + 1 >= arguments.size()) { + throw new AgentRuntimeException("Command file option requires a value."); + } + return arguments.get(optionIndex + 1); + } + + private String attachedOrFollowingValue(List arguments, int optionIndex, String shortOption) { + String argument = arguments.get(optionIndex); + if (isAttachedShortOption(argument, shortOption)) { + return argument.substring(shortOption.length()); + } + return optionValue(arguments, optionIndex); + } + + private boolean isAttachedShortOption(String argument, String option) { + return argument.startsWith(option) && argument.length() > option.length() + && !argument.startsWith("--"); + } + + private boolean isShortOptionPresent(String argument, char option) { + return argument.startsWith("-") && !argument.startsWith("--") + && argument.length() > 1 && argument.substring(1).indexOf(option) >= 0; + } + + private boolean containsUnsafeSubstitutionFlag(String expression) { + for (int index = 0; index + 1 < expression.length(); index++) { + if (expression.charAt(index) != 's' || Character.isLetterOrDigit(expression.charAt(index + 1))) { + continue; + } + char delimiter = expression.charAt(index + 1); + int patternEnd = findUnescaped(expression, delimiter, index + 2); + if (patternEnd < 0) { + continue; + } + int replacementEnd = findUnescaped(expression, delimiter, patternEnd + 1); + if (replacementEnd < 0) { + continue; + } + for (int flagIndex = replacementEnd + 1; flagIndex < expression.length(); flagIndex++) { + char flag = expression.charAt(flagIndex); + if (flag == ';' || flag == '\n' || flag == '}') { + break; + } + if (flag == 'e' || flag == 'w' || flag == 'W') { + return true; + } + if (!Character.isWhitespace(flag) && !Character.isDigit(flag) + && "gIpMm".indexOf(flag) < 0) { + break; + } + } + } + return false; + } + + private int findUnescaped(String value, char delimiter, int start) { + boolean escaped = false; + for (int index = start; index < value.length(); index++) { + char current = value.charAt(index); + if (escaped) { + escaped = false; + } else if (current == '\\') { + escaped = true; + } else if (current == delimiter) { + return index; + } + } + return -1; + } + + private boolean isSafeSedFlag(String argument) { + if (Set.of("-n", "--quiet", "--silent", "-E", "-r", "--regexp-extended", "--sandbox") + .contains(argument)) { + return true; + } + return argument.matches("-[nEr]+"); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupport.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupport.java new file mode 100644 index 0000000..e3c98b3 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupport.java @@ -0,0 +1,150 @@ +package com.easyagents.agent.runtime.tool.operate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Linux Shell 独立会话与进程组清理支持。 + * + *

Linux 使用受信任的 util-linux {@code setsid} 创建独立会话,并通过系统 {@code kill} + * 向负 PGID 发送信号。JDK 17 没有可移植的 killpg API,非 Linux 平台保留 ProcessHandle + * 后代跟踪降级;脚本显式创建第二个会话仍属于无 OS 沙箱时无法消除的边界。 + */ +final class ShellProcessGroupSupport { + + private static final Logger logger = LoggerFactory.getLogger(ShellProcessGroupSupport.class); + private static final List SETSID_CANDIDATES = List.of( + Path.of("/usr/bin/setsid"), Path.of("/bin/setsid")); + private static final List KILL_CANDIDATES = List.of( + Path.of("/bin/kill"), Path.of("/usr/bin/kill")); + + private final Path setsid; + private final Path kill; + + private ShellProcessGroupSupport(Path setsid, Path kill) { + this.setsid = setsid; + this.kill = kill; + } + + /** + * 检测当前平台的进程组能力。 + * + * @return Linux 进程组支持或可移植降级实例 + */ + static ShellProcessGroupSupport detect() { + String osName = System.getProperty("os.name", ""); + if (!isLinux(osName)) { + return new ShellProcessGroupSupport(null, null); + } + return detect(osName, firstExecutable(SETSID_CANDIDATES), firstExecutable(KILL_CANDIDATES)); + } + + /** + * 使用显式路径检测平台能力,供启动校验测试使用。 + * + * @param osName 操作系统名称 + * @param setsidPath setsid 路径,可空 + * @param killPath kill 路径,可空 + * @return 检测结果 + */ + static ShellProcessGroupSupport detect(String osName, Path setsidPath, Path killPath) { + if (!isLinux(osName)) { + return new ShellProcessGroupSupport(null, null); + } + if (!isTrustedExecutable(setsidPath) || !isTrustedExecutable(killPath)) { + throw new WorkspaceToolException("WORKSPACE_CONFIG_INVALID", + "Linux controlled shell requires executable setsid and kill utilities.", false); + } + return new ShellProcessGroupSupport(setsidPath.toAbsolutePath().normalize(), + killPath.toAbsolutePath().normalize()); + } + + /** + * 返回是否启用 Linux 独立进程组。 + * + * @return 启用时为 true + */ + boolean enabled() { + return setsid != null && kill != null; + } + + /** + * 为 Linux 命令增加受信任 setsid 前缀。 + * + * @param command 已校验命令参数 + * @return 实际 ProcessBuilder 参数 + */ + List wrap(List command) { + if (!enabled()) { + return command; + } + List wrapped = new ArrayList<>(command.size() + 1); + wrapped.add(setsid.toString()); + wrapped.addAll(command); + return wrapped; + } + + /** + * 对独立进程组发送 TERM,随后发送 KILL 清理残留成员。 + * + * @param processGroupId setsid 进程 PID,同时也是 PGID + */ + void terminate(long processGroupId) { + if (!enabled() || processGroupId <= 1) { + return; + } + if (!signal("-TERM", processGroupId)) { + return; + } + try { + Thread.sleep(100); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + signal("-KILL", processGroupId); + } + + private boolean signal(String signal, long processGroupId) { + try { + Process process = new ProcessBuilder( + kill.toString(), signal, "--", "-" + processGroupId) + .redirectInput(ProcessBuilder.Redirect.from(Path.of("/dev/null").toFile())) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start(); + if (!process.waitFor(500, TimeUnit.MILLISECONDS)) { + process.destroyForcibly(); + return false; + } + return process.exitValue() == 0; + } catch (IOException error) { + logger.error("Failed to signal controlled shell process group", error); + return false; + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + logger.warn("Interrupted while signaling controlled shell process group", error); + return false; + } + } + + private static boolean isLinux(String osName) { + return osName != null && osName.toLowerCase(Locale.ROOT).contains("linux"); + } + + private static Path firstExecutable(List candidates) { + return candidates.stream().filter(ShellProcessGroupSupport::isTrustedExecutable) + .findFirst().orElse(null); + } + + private static boolean isTrustedExecutable(Path path) { + return path != null && path.isAbsolute() && Files.isRegularFile(path) && Files.isExecutable(path); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/UnifiedPatchParser.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/UnifiedPatchParser.java new file mode 100644 index 0000000..032072b --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/UnifiedPatchParser.java @@ -0,0 +1,287 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import com.easyagents.agent.runtime.tool.operate.ApplyPatchTool.DiffLine; +import com.easyagents.agent.runtime.tool.operate.ApplyPatchTool.FilePatch; +import com.easyagents.agent.runtime.tool.operate.ApplyPatchTool.Hunk; +import com.easyagents.agent.runtime.tool.operate.ApplyPatchTool.PatchType; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * `*** Begin Patch` 和标准 unified diff 解析器。 + */ +final class UnifiedPatchParser { + + private static final Pattern HUNK_HEADER = Pattern.compile( + "^@@(?:\\s+-(\\d+)(?:,\\d+)?\\s+\\+\\d+(?:,\\d+)?\\s+@@.*)?$"); + + private UnifiedPatchParser() { + } + + /** + * 解析补丁文本。 + * + * @param patch 补丁文本 + * @return 有序文件补丁 + */ + static List parse(String patch) { + String normalized = patch.replace("\r\n", "\n").replace('\r', '\n'); + List lines = List.of(normalized.split("\n", -1)); + if (!lines.isEmpty() && "*** Begin Patch".equals(lines.get(0))) { + return parseEnvelope(lines); + } + return parseUnified(lines); + } + + /** + * 将单文件补丁应用到当前文本。 + * + * @param patch 单文件补丁 + * @param current 当前 UTF-8 文本 + * @return 修改后文本 + */ + static String apply(FilePatch patch, String current) { + boolean trailingNewline = patch.type() == PatchType.ADD || current.endsWith("\n") || current.endsWith("\r"); + List content = splitDocument(current); + if (patch.type() == PatchType.DELETE && patch.hunks().isEmpty()) { + return ""; + } + for (Hunk hunk : patch.hunks()) { + List oldLines = hunk.lines().stream() + .filter(line -> line.kind() != '+') + .map(DiffLine::text) + .toList(); + List newLines = hunk.lines().stream() + .filter(line -> line.kind() != '-') + .map(DiffLine::text) + .toList(); + int position = locateUnique(content, oldLines, hunk.oldStart()); + for (int index = 0; index < oldLines.size(); index++) { + if (!content.get(position + index).equals(oldLines.get(index))) { + throw new WorkspaceToolException("PATCH_CONFLICT", + "Patch hunk context does not match the target file.", false); + } + } + content.subList(position, position + oldLines.size()).clear(); + content.addAll(position, newLines); + } + String result = String.join("\n", content); + if (patch.type() == PatchType.DELETE && !result.isEmpty()) { + throw new WorkspaceToolException("PATCH_CONFLICT", + "Delete patch does not match the complete target file.", false); + } + return trailingNewline && !content.isEmpty() ? result + "\n" : result; + } + + private static List parseEnvelope(List lines) { + List patches = new ArrayList<>(); + Set targets = new LinkedHashSet<>(); + int index = 1; + while (index < lines.size()) { + String line = lines.get(index); + if ("*** End Patch".equals(line)) { + return patches; + } + PatchType type; + String path; + if (line.startsWith("*** Add File: ")) { + type = PatchType.ADD; + path = line.substring("*** Add File: ".length()).trim(); + } else if (line.startsWith("*** Update File: ")) { + type = PatchType.UPDATE; + path = line.substring("*** Update File: ".length()).trim(); + } else if (line.startsWith("*** Delete File: ")) { + type = PatchType.DELETE; + path = line.substring("*** Delete File: ".length()).trim(); + } else if (line.isEmpty()) { + index++; + continue; + } else { + throw patchInvalid("Invalid patch section header."); + } + if (path.isBlank() || !targets.add(path)) { + throw patchInvalid("Patch target is empty or duplicated."); + } + index++; + List body = new ArrayList<>(); + while (index < lines.size() && !lines.get(index).startsWith("*** ")) { + body.add(lines.get(index++)); + } + if (!body.isEmpty() && body.get(body.size() - 1).isEmpty()) { + body.remove(body.size() - 1); + } + patches.add(buildFilePatch(type, path, body)); + } + throw patchInvalid("Patch is missing *** End Patch."); + } + + private static List parseUnified(List lines) { + List patches = new ArrayList<>(); + Set targets = new LinkedHashSet<>(); + int index = 0; + while (index < lines.size()) { + if (!lines.get(index).startsWith("--- ")) { + if (lines.get(index).isEmpty()) { + index++; + continue; + } + throw patchInvalid("Invalid unified diff: expected '---' header."); + } + String oldPath = headerPath(lines.get(index++).substring(4)); + if (index >= lines.size() || !lines.get(index).startsWith("+++ ")) { + throw patchInvalid("Invalid unified diff: expected '+++' header."); + } + String newPath = headerPath(lines.get(index++).substring(4)); + PatchType type = "/dev/null".equals(oldPath) ? PatchType.ADD + : "/dev/null".equals(newPath) ? PatchType.DELETE : PatchType.UPDATE; + String path = type == PatchType.DELETE ? stripPrefix(oldPath) : stripPrefix(newPath); + if (path.isBlank() || !targets.add(path)) { + throw patchInvalid("Patch target is empty or duplicated."); + } + List body = new ArrayList<>(); + while (index < lines.size() && !lines.get(index).startsWith("--- ")) { + body.add(lines.get(index++)); + } + if (!body.isEmpty() && body.get(body.size() - 1).isEmpty()) { + body.remove(body.size() - 1); + } + patches.add(buildFilePatch(type, path, body)); + } + return patches; + } + + private static FilePatch buildFilePatch(PatchType type, String path, List body) { + if (type == PatchType.DELETE && body.isEmpty()) { + return new FilePatch(type, path, List.of(), 0, 0); + } + if (type == PatchType.ADD && body.stream().noneMatch(line -> line.startsWith("@@"))) { + List lines = new ArrayList<>(); + for (String line : body) { + if (!line.startsWith("+")) { + throw patchInvalid("Added file lines must start with '+'."); + } + lines.add(new DiffLine('+', line.substring(1))); + } + return new FilePatch(type, path, List.of(new Hunk(1, lines)), lines.size(), 0); + } + List hunks = new ArrayList<>(); + List current = null; + Integer oldStart = null; + int added = 0; + int deleted = 0; + for (String line : body) { + Matcher header = HUNK_HEADER.matcher(line); + if (header.matches()) { + if (current != null) { + hunks.add(new Hunk(oldStart, List.copyOf(current))); + } + current = new ArrayList<>(); + oldStart = header.group(1) == null ? null : Integer.parseInt(header.group(1)); + continue; + } + if ("\\ No newline at end of file".equals(line)) { + continue; + } + if (current == null) { + throw patchInvalid("Patch hunk is missing an @@ header."); + } + if (line.isEmpty() || (line.charAt(0) != ' ' && line.charAt(0) != '+' && line.charAt(0) != '-')) { + throw patchInvalid("Invalid patch hunk line."); + } + char kind = line.charAt(0); + current.add(new DiffLine(kind, line.substring(1))); + if (kind == '+') { + added++; + } else if (kind == '-') { + deleted++; + } + } + if (current != null) { + hunks.add(new Hunk(oldStart, List.copyOf(current))); + } + if (hunks.isEmpty() && type != PatchType.DELETE) { + throw patchInvalid("Patch file section does not contain a hunk."); + } + return new FilePatch(type, path, List.copyOf(hunks), added, deleted); + } + + private static int locateUnique(List content, List oldLines, Integer declaredStart) { + if (oldLines.isEmpty()) { + if (declaredStart == null) { + if (content.isEmpty()) { + return 0; + } + throw new WorkspaceToolException("PATCH_CONFLICT", + "Insertion hunk needs a line position or context.", false); + } + int position = Math.max(0, declaredStart - 1); + if (position > content.size()) { + throw new WorkspaceToolException("PATCH_CONFLICT", + "Insertion position is outside the target file.", false); + } + return position; + } + int match = -1; + for (int start = 0; start + oldLines.size() <= content.size(); start++) { + boolean equal = true; + for (int offset = 0; offset < oldLines.size(); offset++) { + if (!content.get(start + offset).equals(oldLines.get(offset))) { + equal = false; + break; + } + } + if (equal) { + if (match >= 0) { + throw new WorkspaceToolException("PATCH_CONFLICT", + "Patch hunk context is not unique.", false); + } + match = start; + } + } + if (match < 0) { + throw new WorkspaceToolException("PATCH_CONFLICT", + "Patch hunk context was not found.", false); + } + return match; + } + + private static List splitDocument(String content) { + if (content.isEmpty()) { + return new ArrayList<>(); + } + String normalized = content.replace("\r\n", "\n").replace('\r', '\n'); + String[] values = normalized.split("\n", -1); + int length = values.length; + if (length > 0 && values[length - 1].isEmpty()) { + length--; + } + List lines = new ArrayList<>(length); + for (int index = 0; index < length; index++) { + lines.add(values[index]); + } + return lines; + } + + private static String headerPath(String header) { + String trimmed = header.trim(); + int tab = trimmed.indexOf('\t'); + return tab < 0 ? trimmed : trimmed.substring(0, tab); + } + + private static String stripPrefix(String path) { + if (path.startsWith("a/") || path.startsWith("b/")) { + return path.substring(2); + } + return path; + } + + private static WorkspaceToolException patchInvalid(String message) { + return new WorkspaceToolException("PATCH_INVALID", message, false); + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuard.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuard.java new file mode 100644 index 0000000..691ee32 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuard.java @@ -0,0 +1,311 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.regex.Pattern; + +/** + * 工作区路径安全边界。 + * + *

调用方只能提交工作区相对路径。该类拒绝路径穿越、宿主绝对路径、符号链接、设备文件和 + * 其他非普通文件目标,并只向上层返回相对展示路径。 + */ +public final class WorkspacePathGuard { + + private static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("^[A-Za-z]:[\\\\/].*"); + private final Path workspaceRoot; + + /** + * 创建路径保护器并确保工作区根目录存在。 + * + * @param workspaceRoot 受信任的工作区绝对目录 + * @throws AgentRuntimeException 根目录无效或无法创建时抛出 + */ + public WorkspacePathGuard(Path workspaceRoot) { + if (workspaceRoot == null || !workspaceRoot.isAbsolute()) { + throw new WorkspaceToolException("WORKSPACE_CONFIG_INVALID", + "Workspace root must be an absolute path.", false); + } + try { + Files.createDirectories(workspaceRoot.normalize()); + this.workspaceRoot = workspaceRoot.normalize().toRealPath(); + if (!Files.isDirectory(this.workspaceRoot, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("WORKSPACE_CONFIG_INVALID", + "Workspace root is not a directory.", false); + } + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_CONFIG_INVALID", + "Workspace root cannot be initialized.", false, error); + } + } + + /** + * 获取仅供受信任 Runtime 内部使用的真实工作区根目录。 + * + * @return 真实工作区根目录 + */ + Path root() { + return workspaceRoot; + } + + /** + * 解析已存在的普通文件。 + * + * @param relativePath 模型提交的工作区相对路径 + * @return 受控普通文件路径 + * @throws AgentRuntimeException 路径不安全、目标不存在或不是普通文件时抛出 + */ + public Path resolveExistingFile(String relativePath) { + Path target = resolve(relativePath, false); + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_NOT_FOUND", "Workspace file does not exist.", false); + } + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Workspace target is not a regular file.", false); + } + rejectHardLink(target); + return target; + } + + /** + * 解析已存在的普通文件或目录,用于受控命令参数预检。 + * + * @param relativePath 模型提交的工作区相对路径 + * @return 受控现有条目 + * @throws AgentRuntimeException 目标不安全、不存在或属于特殊文件时抛出 + */ + public Path resolveExistingEntry(String relativePath) { + Path target = resolve(relativePath, true); + if (Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + rejectHardLink(target); + return target; + } + if (Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS)) { + return target; + } + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Workspace target is not a regular file or directory.", false); + } + + /** + * 解析命令声明的工作区路径,允许尚不存在的创建目标和已存在的普通文件或目录。 + * + * @param relativePath 命令路径参数 + * @return 受控工作区路径 + * @throws AgentRuntimeException 路径越界、包含链接或属于特殊文件时抛出 + */ + Path resolveCommandPath(String relativePath) { + Path target = resolve(relativePath, true); + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + return target; + } + if (Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + rejectHardLink(target); + return target; + } + if (Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS)) { + return target; + } + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Shell target is not a regular file or directory.", false); + } + + /** + * 解析已存在的目录。 + * + * @param relativePath 模型提交的工作区相对路径,`.` 表示工作区根 + * @return 受控目录路径 + * @throws AgentRuntimeException 路径不安全、目标不存在或不是目录时抛出 + */ + public Path resolveExistingDirectory(String relativePath) { + Path target = resolve(relativePath, true); + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_NOT_FOUND", "Workspace directory does not exist.", false); + } + if (!Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Workspace target is not a directory.", false); + } + return target; + } + + /** + * 解析可写入的文件路径,允许目标和父目录尚未创建。 + * + * @param relativePath 模型提交的工作区相对路径 + * @return 受控文件路径 + * @throws AgentRuntimeException 路径不安全或现有目标不是普通文件时抛出 + */ + public Path resolveForWrite(String relativePath) { + Path target = resolve(relativePath, false); + if (target.equals(workspaceRoot)) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Workspace root cannot be used as a file target.", false); + } + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Workspace target is not a regular file.", false); + } + if (Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + rejectHardLink(target); + } + return target; + } + + /** + * 安全创建目标文件的父目录。 + * + * @param target 已由本保护器解析的目标路径 + * @throws AgentRuntimeException 父目录创建失败或出现符号链接时抛出 + */ + public void ensureParentDirectories(Path target) { + requireInsideWorkspace(target); + Path parent = target.getParent(); + if (parent == null || parent.equals(workspaceRoot)) { + return; + } + Path relative = workspaceRoot.relativize(parent); + Path current = workspaceRoot; + try { + for (Path segment : relative) { + current = current.resolve(segment); + if (Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { + rejectSymbolicLink(current); + if (!Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Workspace parent is not a directory.", false); + } + continue; + } + Files.createDirectory(current); + rejectSymbolicLink(current); + } + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace parent directory cannot be created.", true, error); + } + } + + /** + * 再次校验目标路径的现有链路不包含符号链接,供原子提交前缩短竞态窗口。 + * + * @param target 已解析目标 + * @throws AgentRuntimeException 路径越界或包含符号链接时抛出 + */ + public void revalidate(Path target) { + requireInsideWorkspace(target); + rejectExistingSymbolicLinks(target); + if (Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + rejectHardLink(target); + } + } + + /** + * 将内部路径转换为不泄露宿主目录的工作区相对展示路径。 + * + * @param target 工作区内路径 + * @return 使用正斜杠的相对路径,根目录返回 `.` + */ + public String display(Path target) { + requireInsideWorkspace(target); + Path relative = workspaceRoot.relativize(target.normalize()); + if (relative.toString().isEmpty()) { + return "."; + } + return relative.toString().replace(target.getFileSystem().getSeparator(), "/"); + } + + private Path resolve(String relativePath, boolean allowRoot) { + validateRelativeInput(relativePath, allowRoot); + Path submitted; + try { + submitted = Path.of(relativePath); + } catch (RuntimeException error) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", "Invalid workspace path.", false, error); + } + Path target = workspaceRoot.resolve(submitted).normalize(); + requireInsideWorkspace(target); + rejectExistingSymbolicLinks(target); + return target; + } + + private void validateRelativeInput(String relativePath, boolean allowRoot) { + if (relativePath == null || relativePath.isBlank() || relativePath.indexOf('\0') >= 0) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Workspace path is required and must not contain NUL.", false); + } + String trimmed = relativePath.trim(); + if (trimmed.startsWith("~") || WINDOWS_ABSOLUTE_PATH.matcher(trimmed).matches()) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Only workspace-relative paths are allowed.", false); + } + Path submitted; + try { + submitted = Path.of(trimmed); + } catch (RuntimeException error) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", "Invalid workspace path.", false, error); + } + if (submitted.isAbsolute()) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Only workspace-relative paths are allowed.", false); + } + for (Path segment : submitted) { + if ("..".equals(segment.toString())) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Workspace path traversal is not allowed.", false); + } + } + if (!allowRoot && (".".equals(trimmed) || submitted.getNameCount() == 0)) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Workspace root cannot be used as a file target.", false); + } + } + + private void rejectExistingSymbolicLinks(Path target) { + Path relative = workspaceRoot.relativize(target); + Path current = workspaceRoot; + for (Path segment : relative) { + current = current.resolve(segment); + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { + break; + } + rejectSymbolicLink(current); + } + } + + private void rejectSymbolicLink(Path path) { + if (Files.isSymbolicLink(path)) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Symbolic links are not allowed in workspace paths.", false); + } + } + + private void rejectHardLink(Path path) { + try { + Object value = Files.getAttribute(path, "unix:nlink", LinkOption.NOFOLLOW_LINKS); + if (value instanceof Number number && number.longValue() > 1) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Hard-linked files are not allowed in workspace paths.", false); + } + } catch (UnsupportedOperationException ignored) { + // 非 Unix 文件系统没有 unix:nlink 属性,仍保留 NOFOLLOW 与普通文件类型校验。 + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace file link count cannot be inspected.", true, error); + } + } + + private void requireInsideWorkspace(Path target) { + if (target == null || !target.normalize().startsWith(workspaceRoot)) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Workspace path escapes the configured root.", false); + } + } + +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaGuard.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaGuard.java new file mode 100644 index 0000000..545f9ed --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaGuard.java @@ -0,0 +1,263 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Map; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; + +/** + * 工作区容量与文件数量校验器。 + */ +final class WorkspaceQuotaGuard { + + private static final long MAX_SCANNED_ENTRIES = 100_000L; + private static final int DEFAULT_ARCHIVE_ENTRY_LIMIT = 10_000; + private static final long DEFAULT_ARCHIVE_TOTAL_LIMIT = 512L * 1024L * 1024L; + private static final long DEFAULT_ARCHIVE_FILE_LIMIT = 64L * 1024L * 1024L; + + private final WorkspacePathGuard pathGuard; + private final WorkspaceQuotaLimits limits; + private final WorkspaceQuotaHook hook; + + /** + * 创建配额校验器。 + * + * @param pathGuard 路径保护器 + * @param limits 配额限制 + * @param hook 业务侧附加校验 Hook + */ + WorkspaceQuotaGuard(WorkspacePathGuard pathGuard, + WorkspaceQuotaLimits limits, + WorkspaceQuotaHook hook) { + this.pathGuard = pathGuard; + this.limits = limits == null ? WorkspaceQuotaLimits.unlimited() : limits; + this.hook = hook == null ? WorkspaceQuotaHook.noop() : hook; + } + + /** + * 校验文件是否允许被完整读取。 + * + * @param target 目标普通文件 + */ + void validateFullRead(Path target) { + try { + long size = Files.size(target); + if (limits.getMaxReadSize() > 0 && size > limits.getMaxReadSize()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace full-file read exceeds max-read-size.", false); + } + hook.beforeRead(pathGuard.root(), target, size); + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace file size cannot be inspected.", true, error); + } + } + + /** + * 记录一次范围读取并调用业务侧配额 Hook。 + * + * @param target 目标文件 + * @param readBytes 实际返回字节数 + */ + void validateRangeRead(Path target, long readBytes) { + if (limits.getMaxReadSize() > 0 && readBytes > limits.getMaxReadSize()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace range read exceeds max-read-size.", false); + } + hook.beforeRead(pathGuard.root(), target, readBytes); + } + + /** + * 获取范围读取字节上限。 + * + * @return 字节上限,零表示使用 Runtime 固定安全上限 + */ + long maxReadSize() { + return limits.getMaxReadSize() > 0 ? limits.getMaxReadSize() : 2L * 1024L * 1024L; + } + + /** + * 获取单层目录最大返回条目数。 + * + * @return 最大条目数 + */ + int maxDirectoryEntries() { + long configured = limits.getMaxFileCount(); + return configured > 0 ? (int) Math.min(configured, 1000) : 1000; + } + + /** + * 获取安全归档单次最大条目数。 + * + * @return 条目数上限 + */ + int maxArchiveEntries() { + long configured = limits.getMaxFileCount(); + return configured > 0 + ? (int) Math.min(configured, DEFAULT_ARCHIVE_ENTRY_LIMIT) + : DEFAULT_ARCHIVE_ENTRY_LIMIT; + } + + /** + * 获取安全归档展开总量上限。 + * + * @return 展开总字节数上限 + */ + long maxArchiveTotalSize() { + long configured = limits.getMaxTotalSize(); + return configured > 0 ? Math.min(configured, DEFAULT_ARCHIVE_TOTAL_LIMIT) : DEFAULT_ARCHIVE_TOTAL_LIMIT; + } + + /** + * 获取安全归档单文件上限。 + * + * @return 单文件字节数上限 + */ + long maxArchiveSingleFileSize() { + long configured = limits.getMaxSingleFileSize(); + return configured > 0 ? Math.min(configured, DEFAULT_ARCHIVE_FILE_LIMIT) : DEFAULT_ARCHIVE_FILE_LIMIT; + } + + /** + * 校验单个文件变更后的工作区配额。 + * + * @param target 目标文件 + * @param resultingBytes 变更后的文件字节数,删除时为零 + */ + void validateWrite(Path target, long resultingBytes) { + validateBatch(Map.of(target, resultingBytes)); + } + + /** + * 校验一批文件变更后的工作区配额。 + * + * @param resultingSizes 目标路径到变更后字节数的映射,负数表示删除 + */ + void validateBatch(Map resultingSizes) { + if (resultingSizes == null || resultingSizes.isEmpty()) { + return; + } + WorkspaceUsage usage = scanUsage(); + long projectedSize = usage.totalSize(); + long projectedCount = usage.entryCount(); + Set plannedEntries = new HashSet<>(); + for (Map.Entry entry : resultingSizes.entrySet()) { + Path target = entry.getKey(); + long resultingBytes = entry.getValue() == null ? 0 : entry.getValue(); + boolean exists = Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS); + long previousBytes = sizeIfRegular(target); + projectedSize -= previousBytes; + if (resultingBytes < 0) { + if (exists) { + projectedCount--; + } + hook.beforeWrite(pathGuard.root(), target, previousBytes, 0); + continue; + } + if (limits.getMaxSingleFileSize() > 0 && resultingBytes > limits.getMaxSingleFileSize()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace file exceeds max-single-file-size.", false); + } + try { + projectedSize = Math.addExact(projectedSize, resultingBytes); + } catch (ArithmeticException error) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace exceeds max-total-size.", false, error); + } + if (!exists) { + if (plannedEntries.add(target)) { + projectedCount++; + } + Path parent = target.getParent(); + while (parent != null && !parent.equals(pathGuard.root())) { + if (!Files.exists(parent, LinkOption.NOFOLLOW_LINKS) && plannedEntries.add(parent)) { + projectedCount++; + } + parent = parent.getParent(); + } + } + hook.beforeWrite(pathGuard.root(), target, previousBytes, resultingBytes); + } + if (limits.getMaxTotalSize() > 0 && projectedSize > limits.getMaxTotalSize()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace exceeds max-total-size.", false); + } + if (limits.getMaxFileCount() > 0 && projectedCount > limits.getMaxFileCount()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace exceeds max-file-count.", false); + } + } + + /** + * 校验当前工作区已处于配额范围内。 + */ + void validateCurrentUsage() { + WorkspaceUsage usage = scanUsage(); + if (limits.getMaxTotalSize() > 0 && usage.totalSize() > limits.getMaxTotalSize()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace exceeds max-total-size.", false); + } + if (limits.getMaxFileCount() > 0 && usage.entryCount() > limits.getMaxFileCount()) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace exceeds max-file-count.", false); + } + } + + private WorkspaceUsage scanUsage() { + long totalSize = 0; + long entryCount = 0; + try (Stream paths = Files.walk(pathGuard.root())) { + for (Path path : (Iterable) paths::iterator) { + if (path.equals(pathGuard.root())) { + continue; + } + entryCount++; + if (entryCount > MAX_SCANNED_ENTRIES) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace contains too many entries to inspect safely.", false); + } + if (Files.isSymbolicLink(path)) { + throw new WorkspaceToolException("WORKSPACE_PATH_INVALID", + "Workspace contains a symbolic link.", false); + } + if (Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) { + totalSize = Math.addExact(totalSize, Files.size(path)); + } else if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + throw new WorkspaceToolException("FILE_TYPE_INVALID", + "Workspace contains a non-regular entry.", false); + } + } + return new WorkspaceUsage(totalSize, entryCount); + } catch (IOException | ArithmeticException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace usage cannot be inspected.", true, error); + } + } + + private long sizeIfRegular(Path target) { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + return 0; + } + try { + return Files.size(target); + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace file size cannot be inspected.", true, error); + } + } + + /** + * 工作区当前使用量。 + * + * @param totalSize 普通文件总字节数 + * @param entryCount 文件与目录条目数量,不包含工作区根 + */ + private record WorkspaceUsage(long totalSize, long entryCount) { + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaHook.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaHook.java new file mode 100644 index 0000000..759d9a0 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaHook.java @@ -0,0 +1,74 @@ +package com.easyagents.agent.runtime.tool.operate; + +import java.nio.file.Path; + +/** + * 业务侧可选的工作区配额校验 Hook。 + * + *

Runtime 会先执行内置容量校验,再调用该 Hook。参数中的路径仅供受信任的服务端实现使用, + * 不会进入 Tool Schema、metadata 或模型结果。 + */ +public interface WorkspaceQuotaHook { + + /** + * 在读取普通文件前执行附加校验。 + * + * @param workspaceRoot 工作区根目录 + * @param target 目标普通文件 + * @param requestedBytes 预计读取字节数 + */ + void beforeRead(Path workspaceRoot, Path target, long requestedBytes); + + /** + * 在提交文件变更前执行附加校验。 + * + * @param workspaceRoot 工作区根目录 + * @param target 目标文件 + * @param previousBytes 原文件字节数,不存在时为零 + * @param resultingBytes 新文件字节数,删除时为零 + */ + void beforeWrite(Path workspaceRoot, Path target, long previousBytes, long resultingBytes); + + /** + * 获取无操作 Hook。 + * + * @return 无操作 Hook + */ + static WorkspaceQuotaHook noop() { + return NoopWorkspaceQuotaHook.INSTANCE; + } + + /** + * 无操作 Hook 实现。 + */ + final class NoopWorkspaceQuotaHook implements WorkspaceQuotaHook { + + private static final NoopWorkspaceQuotaHook INSTANCE = new NoopWorkspaceQuotaHook(); + + private NoopWorkspaceQuotaHook() { + } + + /** + * 不执行附加读取校验。 + * + * @param workspaceRoot 工作区根目录 + * @param target 目标普通文件 + * @param requestedBytes 预计读取字节数 + */ + @Override + public void beforeRead(Path workspaceRoot, Path target, long requestedBytes) { + } + + /** + * 不执行附加写入校验。 + * + * @param workspaceRoot 工作区根目录 + * @param target 目标文件 + * @param previousBytes 原文件字节数 + * @param resultingBytes 新文件字节数 + */ + @Override + public void beforeWrite(Path workspaceRoot, Path target, long previousBytes, long resultingBytes) { + } + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaLimits.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaLimits.java new file mode 100644 index 0000000..762d4fb --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceQuotaLimits.java @@ -0,0 +1,78 @@ +package com.easyagents.agent.runtime.tool.operate; + +/** + * 工作区资源配额。 + * + *

所有大小均以字节计。小于等于零的值表示对应维度不限制,便于通用 Runtime 保持兼容, + * 生产系统应由业务侧显式传入有界配置。 + */ +public final class WorkspaceQuotaLimits { + + private final long maxTotalSize; + private final long maxSingleFileSize; + private final long maxFileCount; + private final long maxReadSize; + + /** + * 创建工作区配额。 + * + * @param maxTotalSize 工作区普通文件总字节数 + * @param maxSingleFileSize 单个普通文件最大字节数 + * @param maxFileCount 工作区文件与目录条目最大数量,不包含工作区根 + * @param maxReadSize 单次读取文件最大字节数 + */ + public WorkspaceQuotaLimits(long maxTotalSize, + long maxSingleFileSize, + long maxFileCount, + long maxReadSize) { + this.maxTotalSize = maxTotalSize; + this.maxSingleFileSize = maxSingleFileSize; + this.maxFileCount = maxFileCount; + this.maxReadSize = maxReadSize; + } + + /** + * 创建无限制配额。 + * + * @return 无限制配额 + */ + public static WorkspaceQuotaLimits unlimited() { + return new WorkspaceQuotaLimits(0, 0, 0, 0); + } + + /** + * 获取工作区总量上限。 + * + * @return 总字节数上限 + */ + public long getMaxTotalSize() { + return maxTotalSize; + } + + /** + * 获取单文件上限。 + * + * @return 单文件字节数上限 + */ + public long getMaxSingleFileSize() { + return maxSingleFileSize; + } + + /** + * 获取工作区条目数量上限。 + * + * @return 文件与目录条目数量上限 + */ + public long getMaxFileCount() { + return maxFileCount; + } + + /** + * 获取单次读取上限。 + * + * @return 读取字节数上限 + */ + public long getMaxReadSize() { + return maxReadSize; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceTextFiles.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceTextFiles.java new file mode 100644 index 0000000..2064b7f --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceTextFiles.java @@ -0,0 +1,269 @@ +package com.easyagents.agent.runtime.tool.operate; + +import java.io.IOException; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Set; + +/** + * 工作区 UTF-8 文本文件原子读写辅助方法。 + */ +final class WorkspaceTextFiles { + + private WorkspaceTextFiles() { + } + + /** + * 严格按 UTF-8 读取文件。 + * + * @param target 目标普通文件 + * @return 文件文本 + */ + static String readUtf8(Path target) { + Set options = Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS); + try (SeekableByteChannel channel = Files.newByteChannel(target, options); + java.io.InputStream input = Channels.newInputStream(channel)) { + byte[] bytes = input.readAllBytes(); + return decodeUtf8(bytes); + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace text file cannot be read.", true, error); + } + } + + /** + * 以流式方式读取有界行范围,避免为了返回少量行先加载完整文本。 + * + * @param target 目标普通文件 + * @param ranges 可选行范围,支持 `start,end` 与负数尾部索引 + * @return 带真实起始行号的行范围 + */ + static RangedLines readUtf8Lines(Path target, String ranges, long maxReadBytes) { + ParsedRange range = ParsedRange.parse(ranges); + Set options = Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS); + try (SeekableByteChannel channel = Files.newByteChannel(target, options); + BufferedReader reader = new BufferedReader(new InputStreamReader( + Channels.newInputStream(channel), StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT)))) { + if (range.negative()) { + return readTail(reader, range, maxReadBytes); + } + List selected = new ArrayList<>(); + long selectedBytes = 0; + int lineNumber = 0; + String line; + while ((line = reader.readLine()) != null) { + lineNumber++; + if (lineNumber >= range.start() && lineNumber <= range.end()) { + selectedBytes = addLineBytes(selectedBytes, line, maxReadBytes); + selected.add(line); + } + if (lineNumber >= range.end()) { + break; + } + } + if (lineNumber < range.start() && lineNumber > 0) { + throw invalidRange("Invalid range: start line is outside the file."); + } + return new RangedLines(range.start(), selected, selectedBytes); + } catch (CharacterCodingException error) { + throw new WorkspaceToolException("FILE_ENCODING_INVALID", + "Workspace file is not valid UTF-8 text.", false, error); + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace text file cannot be read.", true, error); + } + } + + /** + * 严格解码 UTF-8 字节。 + * + * @param bytes 文本字节 + * @return UTF-8 文本 + */ + static String decodeUtf8(byte[] bytes) { + try { + CharBuffer decoded = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)); + return decoded.toString(); + } catch (CharacterCodingException error) { + throw new WorkspaceToolException("FILE_ENCODING_INVALID", + "Workspace file is not valid UTF-8 text.", false, error); + } + } + + /** + * 使用同目录临时文件原子替换目标内容。 + * + * @param pathGuard 路径保护器 + * @param target 目标文件 + * @param bytes 新文件字节 + */ + static void atomicWrite(WorkspacePathGuard pathGuard, Path target, byte[] bytes) { + pathGuard.ensureParentDirectories(target); + Path parent = target.getParent(); + Path temporary = null; + try { + temporary = Files.createTempFile(parent, ".easyagents-write-", ".tmp"); + try (FileChannel channel = FileChannel.open( + temporary, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); + } + pathGuard.revalidate(target); + Files.move(temporary, target, + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + forceDirectory(parent); + } catch (IOException error) { + throw new WorkspaceToolException("WORKSPACE_IO_FAILED", + "Workspace text file cannot be committed.", true, error); + } finally { + if (temporary != null) { + try { + Files.deleteIfExists(temporary); + } catch (IOException ignored) { + // 提交失败已经向上抛出,临时文件清理失败由后续工作区清理任务兜底。 + } + } + } + } + + private static RangedLines readTail(BufferedReader reader, + ParsedRange range, + long maxReadBytes) throws IOException { + long requestedKeep = Math.max(Math.abs((long) range.start()), Math.abs((long) range.end())); + if (requestedKeep > Math.min(maxReadBytes, 100_000L)) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Requested tail range exceeds the configured read bound.", false); + } + int keep = Math.toIntExact(requestedKeep); + Deque tail = new ArrayDeque<>(keep); + int lineCount = 0; + String line; + while ((line = reader.readLine()) != null) { + lineCount++; + if (tail.size() == keep) { + tail.removeFirst(); + } + long lineBytes = line.getBytes(StandardCharsets.UTF_8).length + 1L; + if (lineBytes > maxReadBytes) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace range read exceeds max-read-size.", false); + } + tail.addLast(line); + } + if (lineCount == 0) { + return new RangedLines(1, List.of(), 0); + } + int start = Math.max(1, lineCount + range.start() + 1); + int end = Math.min(lineCount, lineCount + range.end() + 1); + if (start > end) { + throw invalidRange("Invalid range: start line is greater than end line."); + } + int retainedStart = lineCount - tail.size() + 1; + List retained = new ArrayList<>(tail); + List selected = new ArrayList<>( + retained.subList(start - retainedStart, end - retainedStart + 1)); + long selectedBytes = 0; + for (String selectedLine : selected) { + selectedBytes = addLineBytes(selectedBytes, selectedLine, maxReadBytes); + } + return new RangedLines(start, selected, selectedBytes); + } + + private static long addLineBytes(long current, String line, long maxReadBytes) { + long updated; + try { + updated = Math.addExact(current, line.getBytes(StandardCharsets.UTF_8).length + 1L); + } catch (ArithmeticException error) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace range read exceeds max-read-size.", false, error); + } + if (updated > maxReadBytes) { + throw new WorkspaceToolException("WORKSPACE_QUOTA_EXCEEDED", + "Workspace range read exceeds max-read-size.", false); + } + return updated; + } + + private static void forceDirectory(Path directory) { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } catch (IOException | UnsupportedOperationException ignored) { + // 某些文件系统不支持目录 fsync;文件内容和原子 rename 已经完成。 + } + } + + private static WorkspaceToolException invalidRange(String message) { + return new WorkspaceToolException("INVALID_ARGUMENT", message, false); + } + + /** + * 流式读取结果。 + * + * @param startLine 第一行真实 1-based 行号 + * @param lines 文本行 + * @param readBytes 返回文本字节数 + */ + record RangedLines(int startLine, List lines, long readBytes) { + } + + /** + * 归一化行范围。 + * + * @param start 起始行,允许负数 + * @param end 结束行,允许负数 + * @param negative 是否为尾部范围 + */ + private record ParsedRange(int start, int end, boolean negative) { + + private static ParsedRange parse(String ranges) { + if (ranges == null || ranges.isBlank()) { + return new ParsedRange(1, Integer.MAX_VALUE, false); + } + String normalized = ranges.trim().replace("[", "").replace("]", ""); + String[] parts = normalized.split(",", -1); + if (parts.length != 2) { + throw invalidRange("Invalid range format. Expected 'start,end'."); + } + try { + int start = Integer.parseInt(parts[0].trim()); + int end = Integer.parseInt(parts[1].trim()); + if (start == 0 || end == 0 || (start < 0) != (end < 0)) { + throw invalidRange("Invalid range: use either positive or negative line numbers."); + } + if (start > end) { + throw invalidRange("Invalid range: start line is greater than end line."); + } + return new ParsedRange(start, end, start < 0); + } catch (NumberFormatException error) { + throw new WorkspaceToolException("INVALID_ARGUMENT", + "Invalid range format. Expected integer line numbers.", false, error); + } + } + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolException.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolException.java new file mode 100644 index 0000000..2f9d935 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolException.java @@ -0,0 +1,57 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; + +/** + * 带稳定工具错误码和重试语义的工作区异常。 + */ +final class WorkspaceToolException extends AgentRuntimeException { + + private final String code; + private final boolean retryable; + + /** + * 创建工具异常。 + * + * @param code 稳定错误码 + * @param message 可安全返回给模型的信息 + * @param retryable 是否可重试 + */ + WorkspaceToolException(String code, String message, boolean retryable) { + super(message); + this.code = code; + this.retryable = retryable; + } + + /** + * 创建带内部原因的工具异常。 + * + * @param code 稳定错误码 + * @param message 可安全返回给模型的信息 + * @param retryable 是否可重试 + * @param cause 仅写入服务端日志的内部原因 + */ + WorkspaceToolException(String code, String message, boolean retryable, Throwable cause) { + super(message, cause); + this.code = code; + this.retryable = retryable; + } + + /** + * 获取稳定错误码。 + * + * @return 错误码 + */ + String code() { + return code; + } + + /** + * 返回是否可重试。 + * + * @return 可重试时为 true + */ + boolean retryable() { + return retryable; + } +} diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolResults.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolResults.java new file mode 100644 index 0000000..a3748e9 --- /dev/null +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/WorkspaceToolResults.java @@ -0,0 +1,58 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import io.agentscope.core.message.ToolResultBlock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 内置工作区工具的稳定错误结果工厂。 + */ +final class WorkspaceToolResults { + + private static final Logger logger = LoggerFactory.getLogger(WorkspaceToolResults.class); + + private WorkspaceToolResults() { + } + + /** + * 将内部异常转换为不含宿主路径的稳定错误对象。 + * + * @param error 内部异常 + * @return Tool 错误结果 + */ + static ToolResultBlock error(AgentRuntimeException error) { + if (error instanceof WorkspaceToolException typed) { + if (typed.getCause() != null) { + logger.error("Workspace tool failed with code {}", typed.code(), typed); + } + return error(typed.code(), typed.getMessage(), typed.retryable()); + } + logger.error("Unexpected workspace tool failure", error); + return error("WORKSPACE_OPERATION_FAILED", "Workspace operation failed.", false); + } + + /** + * 创建稳定错误结果。 + * + * @param code 错误码 + * @param message 安全错误信息 + * @param retryable 是否可重试 + * @return Tool 错误结果 + */ + static ToolResultBlock error(String code, String message, boolean retryable) { + String json = "{\"code\":\"" + escape(code) + "\",\"message\":\"" + + escape(message) + "\",\"retryable\":" + retryable + "}"; + return ToolResultBlock.error(json); + } + + private static String escape(String value) { + if (value == null) { + return ""; + } + return value.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r"); + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java index 62fc4e8..1ee90f9 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java @@ -241,6 +241,7 @@ public class AgentScopeStatefulRuntimeTest { request.getAgentDefinition().setOperateToolSpecs(List.of( operateToolSpec(AgentOperateToolType.READ_FILE), operateToolSpec(AgentOperateToolType.WRITE_FILE), + operateToolSpec(AgentOperateToolType.PATCH), operateToolSpec(AgentOperateToolType.SHELL))); AgentScopeReActRuntime runtime = fakeRuntime(); @@ -251,6 +252,7 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.LIST_DIRECTORY_TOOL)); Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.WRITE_TEXT_FILE_TOOL)); Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.INSERT_TEXT_FILE_TOOL)); + Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.APPLY_PATCH_TOOL)); Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.EXECUTE_SHELL_COMMAND_TOOL)); } @@ -258,14 +260,13 @@ public class AgentScopeStatefulRuntimeTest { public void shouldSuspendShellOperateToolWithToolHitlInterceptor() { AgentInitRequest request = initRequest(); AgentOperateToolSpec shell = operateToolSpec(AgentOperateToolType.SHELL); - shell.setShellAllowedCommands(Set.of()); request.getAgentDefinition().setOperateToolSpecs(List.of(shell)); AgentScopeReActRuntime runtime = runtimeWithModel(List.of(ChatResponse.builder() .id("shell-call-message") .content(List.of(ToolUseBlock.builder() .id("call-shell") .name(AgentOperateToolAdapter.EXECUTE_SHELL_COMMAND_TOOL) - .input(Map.of("command", "echo hello")) + .input(Map.of("command", "pwd")) .build())) .finishReason("tool_calls") .build())); @@ -280,6 +281,36 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertTrue(events.stream().anyMatch(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED)); } + @Test + public void shouldForceApprovalForRemoveWhenShellApprovalIsDisabled() { + AgentInitRequest request = initRequest(); + AgentOperateToolSpec shell = operateToolSpec(AgentOperateToolType.SHELL); + shell.setApprovalRequired(false); + request.getAgentDefinition().setOperateToolSpecs(List.of(shell)); + AgentScopeReActRuntime runtime = runtimeWithModel(List.of(ChatResponse.builder() + .id("forced-remove-message") + .content(List.of(ToolUseBlock.builder() + .id("call-remove") + .name(AgentOperateToolAdapter.EXECUTE_SHELL_COMMAND_TOOL) + .input(Map.of("command", "'rm' removable.txt")) + .build())) + .finishReason("tool_calls") + .build())); + + runtime.init(request); + List events = runtime.stream(AgentMessage.text(AgentMessageRole.USER, "remove file")) + .collectList() + .block(Duration.ofSeconds(5)); + + Assert.assertNotNull(events); + Assert.assertTrue(events.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED)); + Assert.assertTrue(events.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED)); + Assert.assertFalse(events.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_RESULT)); + } + @Test(expected = AgentRuntimeException.class) public void shouldRejectOperateToolNameConflictWithBusinessTool() { AgentInitRequest request = initRequest(); @@ -498,10 +529,12 @@ public class AgentScopeStatefulRuntimeTest { ToolUseBlock toolUse = ToolUseBlock.builder() .id("call-1") .name("search") - .input(Map.of("q", "easyflow")) + .input(Map.of("q", "sentinel-secret-input")) + .metadata(Map.of("authorization", "sentinel-secret-metadata")) .build(); ToolResultBlock toolResult = ToolResultBlock.of("call-1", "search", - TextBlock.builder().text("done").build(), Map.of("success", true)); + TextBlock.builder().text("sentinel-secret-result").build(), + Map.of("success", true, "token", "sentinel-secret-result-metadata")); observer.observe(new PreActingEvent(agent, toolkit, toolUse)).block(); observer.observe(new PostActingEvent(agent, toolkit, toolUse, toolResult)).block(); @@ -509,13 +542,16 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertEquals(AgentRuntimeEventType.TOOL_CALL, events.get(0).getEventType()); Assert.assertEquals("RUNNING", events.get(0).getPayload().get("status")); - Assert.assertEquals("PRE_ACTING", events.get(0).getPayload().get("phase")); Assert.assertEquals("Search Tool", events.get(0).getPayload().get("toolDisplayName")); - Assert.assertEquals("search", events.get(0).getPayload().get("rawMcpToolName")); + Assert.assertFalse(events.get(0).getPayload().containsKey("input")); + Assert.assertFalse(events.get(0).getPayload().containsKey("content")); + Assert.assertFalse(events.get(0).getMetadata().toString().contains("sentinel-secret")); Assert.assertEquals(AgentRuntimeEventType.TOOL_RESULT, events.get(1).getEventType()); Assert.assertEquals("SUCCESS", events.get(1).getPayload().get("status")); - Assert.assertEquals("POST_ACTING", events.get(1).getPayload().get("phase")); Assert.assertEquals("Search Tool", events.get(1).getPayload().get("toolDisplayName")); + Assert.assertFalse(events.get(1).getPayload().containsKey("text")); + Assert.assertFalse(events.get(1).getMetadata().toString().contains("sentinel-secret")); + Assert.assertFalse(events.toString().contains("sentinel-secret")); } @Test @@ -748,7 +784,12 @@ public class AgentScopeStatefulRuntimeTest { @Test public void shouldRejectConcurrentStatefulStream() { - AgentScopeReActRuntime runtime = fakeRuntime(); + AgentScopeReActRuntime runtime = runtimeWithModel(List.of( + ChatResponse.builder() + .id("slow-response") + .content(List.of(TextBlock.builder().text("still running").build())) + .finishReason("stop") + .build()), Duration.ofSeconds(1)); runtime.init(initRequest()); reactor.core.Disposable disposable = runtime.stream(AgentMessage.text(AgentMessageRole.USER, "first")) @@ -974,6 +1015,188 @@ public class AgentScopeStatefulRuntimeTest { Assert.assertTrue(sessionStore.exists("session-1")); } + /** + * 验证同一 Turn 内同一 MCP 的后续工具复用一次批准,新 Turn 会重新请求批准。 + */ + @Test + public void shouldReuseMcpApprovalWithinTurnAndResetForNextTurn() { + AgentInitRequest request = initRequest(); + AgentToolSpec resolveSpec = approvalRequiredMcpTool( + "mcp_101_resolve_library_id", "101"); + AgentToolSpec querySpec = approvalRequiredMcpTool( + "mcp_101_query_docs", "101"); + request.getAgentDefinition().setToolSpecs(List.of(resolveSpec, querySpec)); + AtomicInteger invocationCount = new AtomicInteger(); + request.setToolInvokers(Map.of( + resolveSpec.getName(), (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("library-id"); + }, + querySpec.getName(), (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("docs"); + })); + AgentScopeReActRuntime runtime = runtimeWithSequentialModel(List.of( + toolResponse("resolve-call", "call-resolve", resolveSpec.getName()), + toolResponse("query-call", "call-query", querySpec.getName()), + ChatResponse.builder() + .id("final-message") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build(), + toolResponse("next-turn-call", "call-next", resolveSpec.getName()))); + runtime.init(request); + + List initialEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "介绍 AG-UI")) + .collectList() + .block(); + AgentRuntimeEvent approval = initialEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .findFirst() + .orElseThrow(); + + List resumeEvents = runtime.resume(resumeFromApproval(approval, true)) + .collectList() + .block(); + + Assert.assertEquals(2, invocationCount.get()); + Assert.assertFalse(resumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED)); + Assert.assertTrue(resumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); + + List nextTurnEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "再查一次")) + .collectList() + .block(); + + Assert.assertEquals(1, nextTurnEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .count()); + Assert.assertTrue(nextTurnEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED)); + } + + /** + * 验证同一推理消息中同一 MCP 的多个工具只生成一个审批请求。 + */ + @Test + public void shouldRequestOneApprovalForParallelToolsFromSameMcp() { + AgentInitRequest request = initRequest(); + AgentToolSpec resolveSpec = approvalRequiredMcpTool( + "mcp_101_resolve_library_id", "101"); + AgentToolSpec querySpec = approvalRequiredMcpTool( + "mcp_101_query_docs", "101"); + request.getAgentDefinition().setToolSpecs(List.of(resolveSpec, querySpec)); + AtomicInteger invocationCount = new AtomicInteger(); + request.setToolInvokers(Map.of( + resolveSpec.getName(), (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("library-id"); + }, + querySpec.getName(), (arguments, context) -> { + invocationCount.incrementAndGet(); + return AgentToolResult.success("docs"); + })); + AgentScopeReActRuntime runtime = runtimeWithSequentialModel(List.of( + ChatResponse.builder() + .id("parallel-mcp-tools") + .content(List.of( + ToolUseBlock.builder() + .id("call-resolve") + .name(resolveSpec.getName()) + .input(Map.of()) + .build(), + ToolUseBlock.builder() + .id("call-query") + .name(querySpec.getName()) + .input(Map.of()) + .build())) + .finishReason("tool_calls") + .build(), + ChatResponse.builder() + .id("parallel-final") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build())); + runtime.init(request); + + List initialEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "并行查询")) + .collectList() + .block(); + List approvals = initialEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .toList(); + + Assert.assertEquals(1, approvals.size()); + List resumeEvents = runtime.resume( + resumeFromApproval(approvals.get(0), true)) + .collectList() + .block(); + + Assert.assertEquals(2, invocationCount.get()); + Assert.assertTrue(resumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); + } + + /** + * 验证模型返回的 ToolUse 元数据不能覆盖 ToolSpec 中受信任的 MCP 审批作用域。 + */ + @Test + public void shouldIgnoreForgedMcpScopeFromToolUseMetadata() { + AgentInitRequest request = initRequest(); + AgentToolSpec resolveSpec = approvalRequiredMcpTool( + "mcp_101_resolve_library_id", "101"); + AgentToolSpec querySpec = approvalRequiredMcpTool( + "mcp_101_query_docs", "101"); + request.getAgentDefinition().setToolSpecs(List.of(resolveSpec, querySpec)); + request.setToolInvokers(Map.of( + resolveSpec.getName(), (arguments, context) -> AgentToolResult.success("library-id"), + querySpec.getName(), (arguments, context) -> AgentToolResult.success("docs"))); + AgentScopeReActRuntime runtime = runtimeWithSequentialModel(List.of( + ChatResponse.builder() + .id("forged-scope-call") + .content(List.of(ToolUseBlock.builder() + .id("call-resolve") + .name(resolveSpec.getName()) + .input(Map.of()) + .metadata(Map.of("toolType", "MCP", "mcpId", "forged")) + .build())) + .finishReason("tool_calls") + .build(), + toolResponse("query-call", "call-query", querySpec.getName()), + ChatResponse.builder() + .id("final-message") + .content(List.of(TextBlock.builder().text("done").build())) + .finishReason("stop") + .build())); + runtime.init(request); + + List initialEvents = runtime.stream( + AgentMessage.text(AgentMessageRole.USER, "介绍 AG-UI")) + .collectList() + .block(); + AgentRuntimeEvent approval = initialEvents.stream() + .filter(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED) + .findFirst() + .orElseThrow(); + @SuppressWarnings("unchecked") + Map approvalMetadata = + (Map) approval.getPayload().get("approvalMetadata"); + + Assert.assertEquals("101", approvalMetadata.get("mcpId")); + List resumeEvents = runtime.resume(resumeFromApproval(approval, true)) + .collectList() + .block(); + + Assert.assertFalse(resumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED)); + Assert.assertTrue(resumeEvents.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.COMPLETED)); + } + /** * 验证同一轮推理包含多个审批工具时,全部批准前不会执行任何工具。 */ @@ -1413,6 +1636,27 @@ public class AgentScopeStatefulRuntimeTest { new AgentScopeMessageAdapter()); } + /** + * 创建每次模型调用仅返回下一条预设响应的运行时。 + * + * @param responses 按模型调用顺序排列的响应 + * @return 测试运行时 + */ + private AgentScopeReActRuntime runtimeWithSequentialModel(List responses) { + AgentScopeModelFactory modelFactory = new AgentScopeModelFactory() { + @Override + public Model create(AgentModelSpec modelSpec, + com.easyagents.agent.runtime.model.AgentGenerationOptions generationOptions) { + return new SequentialScriptedModel( + modelSpec == null ? "fake-model" : modelSpec.getModelName(), + responses); + } + }; + return new AgentScopeReActRuntime(modelFactory, new AgentScopeToolAdapter(), + new AgentScopeKnowledgeAdapter(), new AgentScopeMemoryAdapter(), new AgentScopeSkillAdapter(), + new AgentScopeMessageAdapter()); + } + /** * 创建单次模型调用返回多个增量响应的运行时。 * @@ -1450,6 +1694,44 @@ public class AgentScopeStatefulRuntimeTest { return request; } + /** + * 创建需要批准且归属于指定 MCP 的工具定义。 + * + * @param toolName 工具名称 + * @param mcpId MCP 标识 + * @return MCP 工具定义 + */ + private AgentToolSpec approvalRequiredMcpTool(String toolName, String mcpId) { + AgentToolSpec spec = new AgentToolSpec(); + spec.setName(toolName); + spec.setDescription(toolName); + spec.setApprovalRequired(true); + spec.getMetadata().put("toolType", "MCP"); + spec.getMetadata().put("mcpId", mcpId); + spec.getMetadata().put("mcpTitle", "Context7"); + return spec; + } + + /** + * 创建包含一次工具调用的模型响应。 + * + * @param messageId 响应消息标识 + * @param toolCallId 工具调用标识 + * @param toolName 工具名称 + * @return 模型响应 + */ + private ChatResponse toolResponse(String messageId, String toolCallId, String toolName) { + return ChatResponse.builder() + .id(messageId) + .content(List.of(ToolUseBlock.builder() + .id(toolCallId) + .name(toolName) + .input(Map.of()) + .build())) + .finishReason("tool_calls") + .build(); + } + private static class ScriptedModel implements Model { private final String modelName; @@ -1483,6 +1765,56 @@ public class AgentScopeStatefulRuntimeTest { } } + /** + * 每次调用按顺序返回一条响应的测试模型。 + */ + private static class SequentialScriptedModel implements Model { + + private final AtomicInteger invocationIndex = new AtomicInteger(); + private final String modelName; + private final List responses; + + /** + * 创建顺序响应模型。 + * + * @param modelName 模型名称 + * @param responses 按调用顺序排列的响应 + */ + private SequentialScriptedModel(String modelName, List responses) { + this.modelName = modelName; + this.responses = responses; + } + + /** + * 返回当前模型调用对应的单条响应。 + * + * @param messages 输入消息 + * @param toolSchemas 工具定义 + * @param options 生成配置 + * @return 单条响应流 + */ + @Override + public Flux stream(List messages, + List toolSchemas, + GenerateOptions options) { + int index = invocationIndex.getAndIncrement(); + if (index >= responses.size()) { + return Flux.error(new IllegalStateException("No scripted response for invocation " + index)); + } + return Flux.just(responses.get(index)); + } + + /** + * 返回模型名称。 + * + * @return 模型名称 + */ + @Override + public String getModelName() { + return modelName; + } + } + /** * 单次调用按顺序返回全部响应增量的测试模型。 */ diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java index 0accff4..537d507 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java @@ -8,6 +8,9 @@ import org.junit.Test; import java.util.List; import java.util.Set; +import java.nio.file.Files; +import java.nio.file.Path; +import java.io.IOException; /** * 测试 Agent 操作类工具适配器。 @@ -30,7 +33,7 @@ public class AgentOperateToolAdapterTest { } @Test - public void shouldRegisterWriteFileToolsWithDefaultHitlEnabled() { + public void shouldRegisterWriteFileToolsWithDefaultHitlDisabled() { Toolkit toolkit = new Toolkit(); AgentOperateToolSpec spec = spec(AgentOperateToolType.WRITE_FILE); @@ -39,20 +42,42 @@ public class AgentOperateToolAdapterTest { Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.WRITE_TEXT_FILE_TOOL)); Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.INSERT_TEXT_FILE_TOOL)); Assert.assertEquals(2, toolSpecs.size()); - Assert.assertTrue(toolSpecs.stream().allMatch(AgentToolSpec::isApprovalRequired)); + Assert.assertTrue(toolSpecs.stream().noneMatch(AgentToolSpec::isApprovalRequired)); + } + + @Test(expected = AgentRuntimeException.class) + public void shouldRejectEmptyShellWhitelist() { + AgentOperateToolSpec spec = spec(AgentOperateToolType.SHELL); + spec.setShellAllowedCommands(Set.of()); + + adapter.register(List.of(spec), new Toolkit()); } @Test - public void shouldRegisterShellToolWithEmptyWhitelist() { + public void shouldRegisterShellWithForcedRmApprovalMetadata() { Toolkit toolkit = new Toolkit(); AgentOperateToolSpec spec = spec(AgentOperateToolType.SHELL); - spec.setShellAllowedCommands(Set.of()); List toolSpecs = adapter.register(List.of(spec), toolkit); Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.EXECUTE_SHELL_COMMAND_TOOL)); Assert.assertEquals(1, toolSpecs.size()); Assert.assertTrue(toolSpecs.get(0).isApprovalRequired()); + Assert.assertNotNull(toolSpecs.get(0).getApprovalPolicy()); + Assert.assertEquals(List.of("rm"), toolSpecs.get(0).getMetadata().get("forceApprovalCommands")); + Assert.assertFalse(toolSpecs.get(0).getMetadata().containsKey("baseDir")); + } + + @Test + public void shouldRegisterPatchWithDefaultHitlDisabled() { + Toolkit toolkit = new Toolkit(); + AgentOperateToolSpec spec = spec(AgentOperateToolType.PATCH); + + List toolSpecs = adapter.register(List.of(spec), toolkit); + + Assert.assertNotNull(toolkit.getTool(AgentOperateToolAdapter.APPLY_PATCH_TOOL)); + Assert.assertEquals(1, toolSpecs.size()); + Assert.assertFalse(toolSpecs.get(0).isApprovalRequired()); } @Test @@ -95,7 +120,12 @@ public class AgentOperateToolAdapterTest { private AgentOperateToolSpec spec(AgentOperateToolType type) { AgentOperateToolSpec spec = new AgentOperateToolSpec(); spec.setType(type); - spec.setBaseDir(System.getProperty("java.io.tmpdir")); + try { + Path workspace = Files.createTempDirectory("operate-tool-adapter-"); + spec.setBaseDir(workspace.toAbsolutePath().toString()); + } catch (IOException error) { + throw new AssertionError(error); + } return spec; } } diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchToolTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchToolTest.java new file mode 100644 index 0000000..025e1cd --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ApplyPatchToolTest.java @@ -0,0 +1,142 @@ +package com.easyagents.agent.runtime.tool.operate; + +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.ToolCallParam; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +/** + * 测试有界补丁工具。 + */ +public class ApplyPatchToolTest { + + @Test + public void shouldApplyMultiFileAddUpdateDeletePatch() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("existing.md"), "old\n"); + Files.writeString(fixture.root().resolve("remove.md"), "remove\n"); + String patch = """ + *** Begin Patch + *** Update File: existing.md + @@ + -old + +new + *** Add File: created.md + +created + *** Delete File: remove.md + *** End Patch"""; + + ToolResultBlock result = call(fixture.tool(), patch); + + Assert.assertTrue(text(result).contains("3 file(s)")); + Assert.assertEquals("new\n", Files.readString(fixture.root().resolve("existing.md"))); + Assert.assertEquals("created\n", Files.readString(fixture.root().resolve("created.md"))); + Assert.assertFalse(Files.exists(fixture.root().resolve("remove.md"))); + } + + @Test + public void shouldApplyStandardUnifiedDiff() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("standard.txt"), "before\n"); + String patch = """ + --- a/standard.txt + +++ b/standard.txt + @@ -1 +1 @@ + -before + +after + """; + + ToolResultBlock result = call(fixture.tool(), patch); + + Assert.assertTrue(text(result).contains("successfully")); + Assert.assertEquals("after\n", Files.readString(fixture.root().resolve("standard.txt"))); + } + + @Test + public void shouldRejectAmbiguousContextWithoutChangingFile() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("ambiguous.txt"), "same\nother\nsame\n"); + String patch = """ + *** Begin Patch + *** Update File: ambiguous.txt + @@ + -same + +changed + *** End Patch"""; + + ToolResultBlock result = call(fixture.tool(), patch); + + Assert.assertTrue(text(result).contains("PATCH_CONFLICT")); + Assert.assertEquals("same\nother\nsame\n", Files.readString(fixture.root().resolve("ambiguous.txt"))); + } + + @Test + public void shouldRejectPathEscapeBeforeChangingAnyFile() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("safe.txt"), "safe\n"); + String patch = """ + *** Begin Patch + *** Update File: safe.txt + @@ + -safe + +changed + *** Add File: ../escape.txt + +escape + *** End Patch"""; + + ToolResultBlock result = call(fixture.tool(), patch); + + Assert.assertTrue(text(result).contains("WORKSPACE_PATH_INVALID")); + Assert.assertEquals("safe\n", Files.readString(fixture.root().resolve("safe.txt"))); + } + + @Test + public void shouldRejectDeleteDiffThatDoesNotMatchCompleteFile() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("delete.txt"), "expected\nextra\n"); + String patch = """ + --- a/delete.txt + +++ /dev/null + @@ -1 +0,0 @@ + -expected + """; + + ToolResultBlock result = call(fixture.tool(), patch); + + Assert.assertTrue(text(result).contains("PATCH_CONFLICT")); + Assert.assertEquals("expected\nextra\n", Files.readString(fixture.root().resolve("delete.txt"))); + } + + private Fixture fixture() throws IOException { + Path root = Files.createTempDirectory("apply-patch-tool-").toAbsolutePath(); + WorkspacePathGuard pathGuard = new WorkspacePathGuard(root); + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard( + pathGuard, + new WorkspaceQuotaLimits(1024 * 1024, 1024 * 1024, 100, 1024 * 1024), + WorkspaceQuotaHook.noop()); + return new Fixture(root, new ApplyPatchTool(pathGuard, quotaGuard, 1024 * 1024, 10, 1024 * 1024)); + } + + private ToolResultBlock call(ApplyPatchTool tool, String patch) { + return tool.callAsync(ToolCallParam.builder().input(Map.of("patch", patch)).build()).block(); + } + + private String text(ToolResultBlock result) { + return ((TextBlock) result.getOutput().get(0)).getText(); + } + + /** + * Patch 测试夹具。 + * + * @param root 工作区根 + * @param tool Patch 工具 + */ + private record Fixture(Path root, ApplyPatchTool tool) { + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ControlledShellToolTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ControlledShellToolTest.java new file mode 100644 index 0000000..4e20860 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ControlledShellToolTest.java @@ -0,0 +1,347 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.hitl.AgentToolApprovalEvaluation; +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.ToolCallParam; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * 测试受控 Shell 策略与执行边界。 + */ +public class ControlledShellToolTest { + + @Test + public void shouldExecuteAllowlistedCommandWithoutLeakingWorkspaceRoot() throws IOException { + Fixture fixture = fixture(); + + String result = execute(fixture.tool(), Map.of("command", "pwd")); + + Assert.assertTrue(result, result.contains("0")); + Assert.assertFalse(result.contains(fixture.root().toString())); + Assert.assertTrue(result, result.contains(".\n")); + } + + @Test + public void shouldRejectOperatorsExpansionAbsolutePathsAndUnknownCommands() throws IOException { + Fixture fixture = fixture(); + + assertRejected(fixture.tool(), "pwd | cat"); + assertRejected(fixture.tool(), "cat $HOME/secret"); + assertRejected(fixture.tool(), "cat /etc/passwd"); + assertRejected(fixture.tool(), "curl https://example.com"); + assertRejected(fixture.tool(), "pwd\ncat secret"); + } + + @Test + public void shouldRestrictScriptEntrypointsAndDangerousRemove() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("safe.py"), "print('ok')\n"); + + assertRejected(fixture.tool(), "python3 -c 'print(1)'"); + assertRejected(fixture.tool(), "python3 -m http.server"); + assertRejected(fixture.tool(), "node --eval '1+1'"); + assertRejected(fixture.tool(), "rm -rf ."); + assertRejected(fixture.tool(), "rm -r -f output"); + assertRejected(fixture.tool(), "rm --recursive --force output"); + Assert.assertTrue(execute(fixture.tool(), Map.of("command", "python3 safe.py")) + .contains("ok")); + } + + @Test + public void shouldExposeOnlyFixedArchiveCommandSet() throws IOException { + Fixture fixture = fixture(); + + assertRejected(fixture.tool(), "tar --checkpoint-action=exec=sh -cf archive.tar input.txt"); + assertRejected(fixture.tool(), "zip -TT sh archive.zip input.txt"); + Assert.assertTrue(ControlledShellTool.DEFAULT_ALLOWED_COMMANDS.containsAll( + Set.of("gzip", "gunzip", "zip", "unzip", "tar"))); + } + + @Test + public void shouldClassifyReadOnlyWriteConversionAndScriptApproval() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("safe.py"), "print('one')\n"); + Files.writeString(fixture.root().resolve("input.pdf"), "%PDF-1.4\n"); + Files.writeString(fixture.root().resolve("input.md"), "# title\n"); + Files.createDirectory(fixture.root().resolve("output")); + + assertApproval(fixture.tool(), "rg --files .", false, false, null); + assertApproval(fixture.tool(), "tree .", false, false, null); + assertApproval(fixture.tool(), "pdftotext input.pdf -", false, false, null); + assertApproval(fixture.tool(), "mkdir generated", true, false, null); + assertApproval(fixture.tool(), "rm generated.txt", true, true, null); + assertApproval(fixture.tool(), "pandoc input.md -o output/report.docx", true, false, null); + assertApproval(fixture.tool(), "soffice --convert-to pdf --outdir output input.md", + true, false, null); + assertApproval(fixture.tool(), "pdftotext input.pdf output/content.txt", true, false, null); + + AgentToolApprovalEvaluation first = fixture.tool().approvalEvaluation( + Map.of("command", "python3 safe.py --mode report")); + Assert.assertTrue(first.valid()); + Assert.assertTrue(first.approvalRequired()); + Assert.assertFalse(first.forced()); + Assert.assertTrue(first.reusableScope().startsWith("SHELL_SCRIPT:")); + + Files.writeString(fixture.root().resolve("safe.py"), "print('two')\n"); + AgentToolApprovalEvaluation changed = fixture.tool().approvalEvaluation( + Map.of("command", "python3 safe.py --mode report")); + Assert.assertNotEquals(first.reusableScope(), changed.reusableScope()); + } + + @Test + public void shouldRejectUnsafeProductivityCommandOptionsBeforeApproval() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("input.md"), "# title\n"); + Files.writeString(fixture.root().resolve("input.pdf"), "%PDF-1.4\n"); + Files.writeString(fixture.root().resolve("paths.txt"), "input.md\n"); + Files.writeString(fixture.root().resolve("args.txt"), "--empty\n"); + Files.createDirectory(fixture.root().resolve("output")); + + for (String command : new String[]{ + "find .", + "tree -l .", + "du --files0-from=paths.txt", + "pandoc input.md --filter cat -o output/report.docx", + "pandoc input.md -ooutput/report.docx", + "pandoc https://example.com -o output/report.docx", + "soffice --accept=socket --convert-to pdf --outdir output input.md", + "pdftoppm /etc/passwd output/page", + "qpdf @args.txt output/result.pdf"}) { + AgentToolApprovalEvaluation evaluation = fixture.tool().approvalEvaluation( + Map.of("command", command)); + Assert.assertFalse(command, evaluation.valid()); + Assert.assertFalse(command, evaluation.approvalRequired()); + } + } + + @Test + public void shouldEnforceTimeoutAndOutputLimit() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("slow.py"), "import time\ntime.sleep(5)\n"); + Files.writeString(fixture.root().resolve("large.py"), "print('x' * 10000)\n"); + + String timeout = execute(fixture.tool(), Map.of("command", "python3 slow.py", "timeout", 1)); + String truncated = execute(fixture.tool(), Map.of("command", "python3 large.py")); + + Assert.assertTrue(timeout.contains("SHELL_TIMEOUT")); + Assert.assertTrue(truncated.contains("truncated=\"true\"")); + Assert.assertTrue(truncated.contains("OUTPUT_TRUNCATED")); + } + + @Test + public void shouldRejectSecondaryExecutionAndIndirectFileOptions() throws IOException { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("input.txt"), "alpha\n"); + Files.writeString(fixture.root().resolve("input.json"), "{\"value\":1}\n"); + Files.writeString(fixture.root().resolve("paths.txt"), "/etc/passwd\n"); + + for (String command : new String[]{ + "awk 'BEGIN { system(\"id\") }' input.txt", + "awk '{ getline value }' input.txt", + "awk '{ print ENVIRON }' input.txt", + "awk -fprogram.awk input.txt", + "awk --profile=profile.txt '{ print }' input.txt", + "sed -e 'e id' input.txt", + "sed '1r input.txt' input.txt", + "sed 's/alpha/beta/w stolen.txt' input.txt", + "sed -i 's/alpha/beta/' input.txt", + "rg --pre cat alpha .", + "rg --pre-glob '*.txt' alpha .", + "rg --hostname-bin pwd alpha .", + "rg -z alpha .", + "rg -L alpha .", + "rg --follow alpha .", + "ls -RL .", + "grep -Rfpatterns.txt alpha .", + "jq 'env' input.json", + "jq '$ENV' input.json", + "jq -Lmodules '.' input.json", + "sort -ooutput.txt input.txt", + "sort --compress-program=cat input.txt", + "uniq input.txt output.txt", + "file -fpaths.txt", + "sha256sum --check paths.txt", + "wc --files0-from=paths.txt", + "tail --follow=name input.txt", + "cp -L input.txt copied.txt", + "cp --symbolic-link input.txt copied.txt", + "cp -l input.txt copied.txt"}) { + assertRejected(fixture.tool(), command); + } + } + + @Test + public void shouldBestEffortTerminateObservedChildAfterSuccessfulScript() throws Exception { + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("spawn.py"), """ + import pathlib + import subprocess + import sys + import time + child = subprocess.Popen( + [sys.executable, '-c', 'import time; time.sleep(30)'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True) + pathlib.Path('child.pid').write_text(str(child.pid), encoding='utf-8') + time.sleep(0.3) + """); + ProcessHandle child = null; + try { + String result = execute(fixture.tool(), Map.of("command", "python3 spawn.py")); + Assert.assertTrue(result, result.contains("0")); + long pid = Long.parseLong(Files.readString(fixture.root().resolve("child.pid")).trim()); + Optional handle = ProcessHandle.of(pid); + child = handle.orElse(null); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (child != null && child.isAlive() && System.nanoTime() < deadline) { + Thread.sleep(20); + } + Assert.assertTrue("Observed child process must be terminated", child == null || !child.isAlive()); + } finally { + if (child != null && child.isAlive()) { + child.destroyForcibly(); + } + } + } + + @Test + public void shouldTerminateLinuxProcessGroupAfterFastParentExit() throws Exception { + assumeLinuxWithPython(); + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("fast-spawn.py"), """ + import pathlib + import subprocess + import sys + child = subprocess.Popen( + [sys.executable, '-c', 'import time; time.sleep(30)'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + pathlib.Path('fast-child.pid').write_text(str(child.pid), encoding='utf-8') + """); + ProcessHandle child = null; + try { + String result = execute(fixture.tool(), Map.of("command", "python3 fast-spawn.py")); + Assert.assertTrue(result, result.contains("0")); + child = ProcessHandle.of(readPid(fixture.root().resolve("fast-child.pid"))).orElse(null); + assertTerminates(child, "Linux process-group child must be terminated after parent exit"); + } finally { + destroyIfAlive(child); + } + } + + @Test + public void shouldTerminateLinuxProcessGroupOnTimeout() throws Exception { + assumeLinuxWithPython(); + Fixture fixture = fixture(); + Files.writeString(fixture.root().resolve("timeout-spawn.py"), """ + import pathlib + import subprocess + import sys + import time + child = subprocess.Popen( + [sys.executable, '-c', 'import time; time.sleep(30)'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + pathlib.Path('timeout-child.pid').write_text(str(child.pid), encoding='utf-8') + time.sleep(30) + """); + ProcessHandle child = null; + try { + String result = execute(fixture.tool(), Map.of( + "command", "python3 timeout-spawn.py", "timeout", 1)); + Assert.assertTrue(result, result.contains("SHELL_TIMEOUT")); + child = ProcessHandle.of(readPid(fixture.root().resolve("timeout-child.pid"))).orElse(null); + assertTerminates(child, "Linux process-group child must be terminated on timeout"); + } finally { + destroyIfAlive(child); + } + } + + private void assumeLinuxWithPython() { + Assume.assumeTrue(System.getProperty("os.name", "").toLowerCase().contains("linux")); + Assume.assumeTrue(Files.isExecutable(Path.of("/usr/bin/python3")) + || Files.isExecutable(Path.of("/usr/local/bin/python3"))); + } + + private long readPid(Path path) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (!Files.exists(path) && System.nanoTime() < deadline) { + Thread.sleep(10); + } + Assert.assertTrue("Child PID file must be created", Files.exists(path)); + return Long.parseLong(Files.readString(path).trim()); + } + + private void assertTerminates(ProcessHandle child, String message) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (child != null && child.isAlive() && System.nanoTime() < deadline) { + Thread.sleep(20); + } + Assert.assertTrue(message, child == null || !child.isAlive()); + } + + private void destroyIfAlive(ProcessHandle process) { + if (process != null && process.isAlive()) { + process.destroyForcibly(); + } + } + + private Fixture fixture() throws IOException { + Path root = Files.createTempDirectory("controlled-shell-").toAbsolutePath(); + WorkspacePathGuard pathGuard = new WorkspacePathGuard(root); + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard( + pathGuard, + new WorkspaceQuotaLimits(1024 * 1024, 1024 * 1024, 100, 1024 * 1024), + WorkspaceQuotaHook.noop()); + AgentOperateToolSpec spec = new AgentOperateToolSpec(); + spec.setShellDefaultTimeout(Duration.ofSeconds(2)); + spec.setShellMaxTimeout(Duration.ofSeconds(3)); + spec.setShellMaxOutputSize(256); + return new Fixture(root, new ControlledShellTool(pathGuard, quotaGuard, spec)); + } + + private String execute(ControlledShellTool tool, Map input) { + ToolResultBlock result = tool.callAsync(ToolCallParam.builder().input(input).build()).block(); + return ((TextBlock) result.getOutput().get(0)).getText(); + } + + private void assertRejected(ControlledShellTool tool, String command) { + String result = execute(tool, Map.of("command", command)); + Assert.assertTrue(result, result.contains("SHELL_COMMAND_DENIED") + || result.contains("WORKSPACE_PATH_INVALID")); + } + + private void assertApproval(ControlledShellTool tool, + String command, + boolean required, + boolean forced, + String scope) { + AgentToolApprovalEvaluation evaluation = tool.approvalEvaluation(Map.of("command", command)); + Assert.assertTrue(command, evaluation.valid()); + Assert.assertEquals(command, required, evaluation.approvalRequired()); + Assert.assertEquals(command, forced, evaluation.forced()); + Assert.assertEquals(command, scope, evaluation.reusableScope()); + } + + /** + * Shell 测试夹具。 + * + * @param root 工作区根 + * @param tool Shell 工具 + */ + private record Fixture(Path root, ControlledShellTool tool) { + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutorTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutorTest.java new file mode 100644 index 0000000..3fec570 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/SafeArchiveCommandExecutorTest.java @@ -0,0 +1,228 @@ +package com.easyagents.agent.runtime.tool.operate; + +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.ToolCallParam; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * 测试 Java 安全归档执行器的创建、展开与恶意条目边界。 + */ +public class SafeArchiveCommandExecutorTest { + + @Test + public void shouldCreateAndExtractGzipZipAndTarArchives() throws IOException { + Fixture fixture = fixture(new WorkspaceQuotaLimits(16 * 1024 * 1024L, + 8 * 1024 * 1024L, 1000, 1024 * 1024)); + Files.createDirectories(fixture.root().resolve("source/nested")); + Files.writeString(fixture.root().resolve("source/nested/value.txt"), "archive-value\n"); + Files.writeString(fixture.root().resolve("plain.txt"), "plain-value\n"); + Files.writeString(fixture.root().resolve("multi-a.txt"), "a\n"); + Files.writeString(fixture.root().resolve("multi-b.txt"), "b\n"); + + assertSuccess(fixture, "gzip -k plain.txt"); + Files.delete(fixture.root().resolve("plain.txt")); + assertSuccess(fixture, "gunzip -k plain.txt.gz"); + Assert.assertEquals("plain-value\n", Files.readString(fixture.root().resolve("plain.txt"))); + assertSuccess(fixture, "gzip multi-a.txt multi-b.txt"); + Assert.assertFalse(Files.exists(fixture.root().resolve("multi-a.txt"))); + Assert.assertFalse(Files.exists(fixture.root().resolve("multi-b.txt"))); + assertSuccess(fixture, "gunzip multi-a.txt.gz multi-b.txt.gz"); + Assert.assertEquals("a\n", Files.readString(fixture.root().resolve("multi-a.txt"))); + Assert.assertEquals("b\n", Files.readString(fixture.root().resolve("multi-b.txt"))); + + assertSuccess(fixture, "zip -q -r bundle.zip source"); + assertSuccess(fixture, "unzip -q bundle.zip -d zip-output"); + Assert.assertEquals("archive-value\n", + Files.readString(fixture.root().resolve("zip-output/source/nested/value.txt"))); + + assertSuccess(fixture, "tar -czf bundle.tar.gz source"); + String listed = execute(fixture, "tar -tzf bundle.tar.gz"); + Assert.assertTrue(listed, listed.contains("source/nested/value.txt")); + Assert.assertFalse(listed.contains(fixture.root().toString())); + assertSuccess(fixture, "tar -xzf bundle.tar.gz -C tar-output"); + Assert.assertEquals("archive-value\n", + Files.readString(fixture.root().resolve("tar-output/source/nested/value.txt"))); + } + + @Test + public void shouldRejectZipSlipDuplicateAndTargetConflictWithoutPartialOutput() throws IOException { + Fixture fixture = fixture(new WorkspaceQuotaLimits(8 * 1024 * 1024L, + 4 * 1024 * 1024L, 1000, 1024 * 1024)); + createZip(fixture.root().resolve("slip.zip"), + new ZipContent("../escape.txt", "escape")); + createZip(fixture.root().resolve("duplicate.zip"), + new ZipContent("same.txt", "first"), new ZipContent("same.txt", "second")); + createZip(fixture.root().resolve("backslash.zip"), + new ZipContent("..\\escape.txt", "escape")); + createZip(fixture.root().resolve("conflict.zip"), + new ZipContent("first.txt", "first"), new ZipContent("second.txt", "second")); + Files.createDirectories(fixture.root().resolve("output")); + Files.writeString(fixture.root().resolve("output/second.txt"), "existing"); + + Assert.assertTrue(execute(fixture, "unzip slip.zip -d slip-output") + .contains("ARCHIVE_ENTRY_DENIED")); + Assert.assertFalse(Files.exists(fixture.root().getParent().resolve("escape.txt"))); + Assert.assertTrue(execute(fixture, "unzip duplicate.zip -d duplicate-output") + .contains("ARCHIVE_DUPLICATE_ENTRY")); + Assert.assertTrue(execute(fixture, "unzip backslash.zip -d backslash-output") + .contains("ARCHIVE_ENTRY_DENIED")); + Assert.assertTrue(execute(fixture, "unzip conflict.zip -d output") + .contains("ARCHIVE_TARGET_CONFLICT")); + Assert.assertFalse(Files.exists(fixture.root().resolve("output/first.txt"))); + Assert.assertEquals("existing", Files.readString(fixture.root().resolve("output/second.txt"))); + } + + @Test + public void shouldRejectTarLinksDevicesAndFifoEntries() throws IOException { + Fixture fixture = fixture(new WorkspaceQuotaLimits(8 * 1024 * 1024L, + 4 * 1024 * 1024L, 1000, 1024 * 1024)); + createTarSpecial(fixture.root().resolve("link.tar"), "link", (byte) '2'); + createTarSpecial(fixture.root().resolve("hard-link.tar"), "hard-link", (byte) '1'); + createTarSpecial(fixture.root().resolve("device.tar"), "device", (byte) '3'); + createTarSpecial(fixture.root().resolve("fifo.tar"), "fifo", (byte) '6'); + + Assert.assertTrue(execute(fixture, "tar -xf link.tar").contains("ARCHIVE_ENTRY_DENIED")); + Assert.assertTrue(execute(fixture, "tar -xf hard-link.tar").contains("ARCHIVE_ENTRY_DENIED")); + Assert.assertTrue(execute(fixture, "tar -xf device.tar").contains("ARCHIVE_ENTRY_DENIED")); + Assert.assertTrue(execute(fixture, "tar -xf fifo.tar").contains("ARCHIVE_ENTRY_DENIED")); + Assert.assertFalse(Files.exists(fixture.root().resolve("link"))); + Assert.assertFalse(Files.exists(fixture.root().resolve("device"))); + Assert.assertFalse(Files.exists(fixture.root().resolve("fifo"))); + } + + @Test + public void shouldRejectExpandedArchiveBeyondQuota() throws IOException { + Fixture fixture = fixture(new WorkspaceQuotaLimits(512, 8 * 1024, 100, 1024)); + createZip(fixture.root().resolve("bomb.zip"), + new ZipContent("expanded.txt", "0".repeat(4096))); + + String output = execute(fixture, "unzip bomb.zip -d expanded"); + + Assert.assertTrue(output, output.contains("WORKSPACE_QUOTA_EXCEEDED")); + Assert.assertFalse(Files.exists(fixture.root().resolve("expanded"))); + } + + @Test + public void shouldRejectUnknownArchiveOptions() throws IOException { + Fixture fixture = fixture(new WorkspaceQuotaLimits(1024 * 1024, 1024 * 1024, 100, 1024)); + Files.writeString(fixture.root().resolve("input.txt"), "input"); + + for (String command : new String[]{ + "gzip -c input.txt", + "zip -T bundle.zip input.txt", + "unzip -o bundle.zip", + "tar --checkpoint-action=exec=id -cf bundle.tar input.txt", + "tar -xf bundle.tar member.txt"}) { + String output = execute(fixture, command); + Assert.assertTrue(command + " => " + output, output.contains("SHELL_COMMAND_DENIED")); + } + } + + @Test + public void shouldCompensateOutputsWhenCommitFailsAfterFirstMove() throws IOException { + Path root = Files.createTempDirectory("safe-archive-compensation-").toAbsolutePath(); + createZip(root.resolve("two-files.zip"), + new ZipContent("first.txt", "first"), new ZipContent("second.txt", "second")); + WorkspacePathGuard pathGuard = new WorkspacePathGuard(root); + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard( + pathGuard, + new WorkspaceQuotaLimits(1024 * 1024, 1024 * 1024, 100, 1024), + WorkspaceQuotaHook.noop()); + SafeArchiveCommandExecutor executor = new SafeArchiveCommandExecutor( + pathGuard, quotaGuard, 1024, committedCount -> { + if (committedCount == 1) { + throw new IllegalStateException("injected commit failure"); + } + }); + + try { + executor.execute(List.of("unzip", "two-files.zip", "-d", "output"), + System.nanoTime() + TimeUnit.SECONDS.toNanos(5)); + Assert.fail("Expected archive commit failure"); + } catch (WorkspaceToolException expected) { + Assert.assertEquals("ARCHIVE_COMMIT_FAILED", expected.code()); + } + + Assert.assertFalse(Files.exists(root.resolve("output/first.txt"))); + Assert.assertFalse(Files.exists(root.resolve("output/second.txt"))); + Assert.assertFalse(Files.exists(root.resolve("output"))); + } + + private Fixture fixture(WorkspaceQuotaLimits limits) throws IOException { + Path root = Files.createTempDirectory("safe-archive-").toAbsolutePath(); + WorkspacePathGuard pathGuard = new WorkspacePathGuard(root); + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard(pathGuard, limits, WorkspaceQuotaHook.noop()); + AgentOperateToolSpec spec = new AgentOperateToolSpec(); + spec.setShellDefaultTimeout(Duration.ofSeconds(5)); + spec.setShellMaxTimeout(Duration.ofSeconds(10)); + return new Fixture(root, new ControlledShellTool(pathGuard, quotaGuard, spec)); + } + + private void assertSuccess(Fixture fixture, String command) { + String output = execute(fixture, command); + Assert.assertTrue(command + " => " + output, output.contains("0")); + Assert.assertFalse(output.contains(fixture.root().toString())); + } + + private String execute(Fixture fixture, String command) { + ToolResultBlock result = fixture.tool().callAsync(ToolCallParam.builder() + .input(Map.of("command", command)).build()).block(); + return ((TextBlock) result.getOutput().get(0)).getText(); + } + + private void createZip(Path target, ZipContent... contents) throws IOException { + try (ZipArchiveOutputStream zip = new ZipArchiveOutputStream(target)) { + for (ZipContent content : contents) { + byte[] bytes = content.content().getBytes(StandardCharsets.UTF_8); + ZipArchiveEntry entry = new ZipArchiveEntry(content.name()); + entry.setSize(bytes.length); + zip.putArchiveEntry(entry); + zip.write(bytes); + zip.closeArchiveEntry(); + } + zip.finish(); + } + } + + private void createTarSpecial(Path target, String name, byte linkFlag) throws IOException { + try (OutputStream raw = Files.newOutputStream(target); + TarArchiveOutputStream tar = new TarArchiveOutputStream(raw)) { + TarArchiveEntry entry = new TarArchiveEntry(name, linkFlag); + entry.setSize(0); + if (linkFlag == '2') { + entry.setLinkName("../outside"); + } + tar.putArchiveEntry(entry); + tar.closeArchiveEntry(); + tar.finish(); + } + } + + private record ZipContent(String name, String content) { + } + + /** + * 归档测试夹具。 + * + * @param root 工作区根 + * @param tool Shell 工具 + */ + private record Fixture(Path root, ControlledShellTool tool) { + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupportTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupportTest.java new file mode 100644 index 0000000..2103685 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/ShellProcessGroupSupportTest.java @@ -0,0 +1,35 @@ +package com.easyagents.agent.runtime.tool.operate; + +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Path; +import java.util.List; + +/** + * 测试 Linux 独立进程组能力的启动检查与非 Linux 降级。 + */ +public class ShellProcessGroupSupportTest { + + @Test + public void shouldFailFastWhenLinuxProcessGroupDependenciesAreMissing() { + try { + ShellProcessGroupSupport.detect("Linux", null, null); + Assert.fail("Expected missing dependency failure"); + } catch (WorkspaceToolException expected) { + Assert.assertEquals("WORKSPACE_CONFIG_INVALID", expected.code()); + Assert.assertFalse(expected.retryable()); + } + } + + @Test + public void shouldUsePortableFallbackOutsideLinux() { + ShellProcessGroupSupport support = ShellProcessGroupSupport.detect( + "Mac OS X", Path.of("/missing/setsid"), Path.of("/missing/kill")); + List command = List.of("pwd"); + + Assert.assertFalse(support.enabled()); + Assert.assertSame(command, support.wrap(command)); + support.terminate(-1); + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspaceFileToolsTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspaceFileToolsTest.java new file mode 100644 index 0000000..3b2c577 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspaceFileToolsTest.java @@ -0,0 +1,188 @@ +package com.easyagents.agent.runtime.tool.operate; + +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.tool.AgentTool; +import io.agentscope.core.tool.ToolCallParam; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 测试安全读写工具与工作区配额。 + */ +public class WorkspaceFileToolsTest { + + @Test + public void shouldWriteAtomicallyAndReadOnlyRequestedRange() throws IOException { + Fixture fixture = fixture(1024, 1024, 10, 1024); + AgentTool write = new SafeWriteFileTool(fixture.pathGuard(), fixture.quotaGuard()).writeTextFileTool(); + AgentTool read = new SafeReadFileTool(fixture.pathGuard(), fixture.quotaGuard()).viewTextFileTool(); + + ToolResultBlock writeResult = call(write, Map.of( + "file_path", "notes/example.txt", + "content", "first\nsecond\nthird\nfourth\n")); + ToolResultBlock readResult = call(read, Map.of( + "file_path", "notes/example.txt", + "ranges", "2,3")); + + Assert.assertTrue(text(writeResult).contains("successfully")); + Assert.assertTrue(text(readResult).contains("2: second")); + Assert.assertTrue(text(readResult).contains("3: third")); + Assert.assertFalse(text(readResult).contains("1: first")); + Assert.assertFalse(text(readResult).contains(fixture.root().toString())); + try (var files = Files.list(fixture.root().resolve("notes"))) { + Assert.assertTrue(files.noneMatch(path -> path.getFileName().toString().startsWith(".easyagents-write-"))); + } + } + + @Test + public void shouldRejectWriteBeyondQuotaWithoutPartialFile() throws IOException { + Fixture fixture = fixture(5, 5, 1, 5); + AgentTool write = new SafeWriteFileTool(fixture.pathGuard(), fixture.quotaGuard()).writeTextFileTool(); + + ToolResultBlock result = call(write, Map.of("file_path", "too-large.txt", "content", "123456")); + + Assert.assertTrue(text(result).contains("WORKSPACE_QUOTA_EXCEEDED")); + Assert.assertFalse(Files.exists(fixture.root().resolve("too-large.txt"))); + } + + @Test + public void shouldCountCreatedDirectoriesAgainstEntryQuota() throws IOException { + Fixture fixture = fixture(1024, 1024, 1, 1024); + AgentTool write = new SafeWriteFileTool(fixture.pathGuard(), fixture.quotaGuard()).writeTextFileTool(); + + String output = text(call(write, Map.of("file_path", "nested/value.txt", "content", "ok"))); + + Assert.assertTrue(output, output.contains("WORKSPACE_QUOTA_EXCEEDED")); + Assert.assertFalse(Files.exists(fixture.root().resolve("nested"))); + } + + @Test + public void shouldInvokeConfiguredQuotaHook() throws IOException { + Path root = Files.createTempDirectory("workspace-hook-").toAbsolutePath(); + WorkspacePathGuard pathGuard = new WorkspacePathGuard(root); + AtomicInteger writes = new AtomicInteger(); + WorkspaceQuotaHook hook = new WorkspaceQuotaHook() { + @Override + public void beforeRead(Path workspaceRoot, Path target, long requestedBytes) { + } + + @Override + public void beforeWrite(Path workspaceRoot, Path target, long previousBytes, long resultingBytes) { + writes.incrementAndGet(); + } + }; + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard( + pathGuard, new WorkspaceQuotaLimits(1024, 1024, 10, 1024), hook); + AgentTool write = new SafeWriteFileTool(pathGuard, quotaGuard).writeTextFileTool(); + + call(write, Map.of("file_path", "hook.txt", "content", "ok")); + + Assert.assertEquals(1, writes.get()); + } + + @Test + public void shouldRejectDirectoryListingWhenWorkspaceContainsSymlink() throws IOException { + Fixture fixture = fixture(1024, 1024, 10, 1024); + Files.writeString(fixture.root().resolve("safe.txt"), "safe"); + Path outside = Files.createTempFile("workspace-list-outside-", ".txt"); + try { + Files.createSymbolicLink(fixture.root().resolve("blocked"), outside); + } catch (UnsupportedOperationException error) { + return; + } + AgentTool list = new SafeReadFileTool(fixture.pathGuard(), fixture.quotaGuard()).listDirectoryTool(); + + String output = text(call(list, Map.of("dir_path", "."))); + + Assert.assertTrue(output, output.contains("WORKSPACE_PATH_INVALID")); + Assert.assertFalse(output.contains(fixture.root().toString())); + Assert.assertFalse(output.contains(outside.toString())); + } + + @Test + public void shouldReadSmallRangeFromFileLargerThanFullReadLimit() throws IOException { + Fixture fixture = fixture(64 * 1024, 64 * 1024, 10, 16); + Files.writeString(fixture.root().resolve("large.txt"), "first\n" + "x".repeat(4096) + "\n"); + AgentTool read = new SafeReadFileTool(fixture.pathGuard(), fixture.quotaGuard()).viewTextFileTool(); + + String output = text(call(read, Map.of("file_path", "large.txt", "ranges", "1,1"))); + + Assert.assertTrue(output, output.contains("1: first")); + Assert.assertFalse(output.contains("WORKSPACE_QUOTA_EXCEEDED")); + } + + @Test + public void shouldBoundDirectoryListingAndReportTruncation() throws IOException { + Fixture fixture = fixture(64 * 1024, 64 * 1024, 0, 1024); + for (int index = 0; index < 1001; index++) { + Files.writeString(fixture.root().resolve("entry-" + index + ".txt"), "x"); + } + AgentTool list = new SafeReadFileTool(fixture.pathGuard(), fixture.quotaGuard()).listDirectoryTool(); + + String output = text(call(list, Map.of("dir_path", "."))); + + Assert.assertTrue(output, output.contains("Truncated: true; limit=1000")); + Assert.assertEquals(1000, output.lines().filter(line -> line.startsWith("file\t")).count()); + } + + @Test + public void shouldRejectDirectoryListingBeforeSortingOverQuotaWorkspace() throws IOException { + Fixture fixture = fixture(64 * 1024, 64 * 1024, 3, 1024); + for (int index = 0; index < 4; index++) { + Files.createDirectory(fixture.root().resolve("directory-" + index)); + } + AgentTool list = new SafeReadFileTool(fixture.pathGuard(), fixture.quotaGuard()).listDirectoryTool(); + + String output = text(call(list, Map.of("dir_path", "."))); + + Assert.assertTrue(output, output.contains("WORKSPACE_QUOTA_EXCEEDED")); + } + + @Test + public void shouldReturnStableReadErrorCodes() throws IOException { + Fixture fixture = fixture(1024, 1024, 10, 1024); + Files.createDirectory(fixture.root().resolve("folder")); + Files.writeString(fixture.root().resolve("valid.txt"), "ok\n"); + AgentTool read = new SafeReadFileTool(fixture.pathGuard(), fixture.quotaGuard()).viewTextFileTool(); + + Assert.assertTrue(text(call(read, Map.of("file_path", "missing.txt"))).contains("FILE_NOT_FOUND")); + Assert.assertTrue(text(call(read, Map.of("file_path", "folder"))).contains("FILE_TYPE_INVALID")); + Assert.assertTrue(text(call(read, Map.of("file_path", "../outside"))) + .contains("WORKSPACE_PATH_INVALID")); + Assert.assertTrue(text(call(read, Map.of("file_path", "valid.txt", "ranges", "x,y"))) + .contains("INVALID_ARGUMENT")); + } + + private Fixture fixture(long total, long single, long count, long read) throws IOException { + Path root = Files.createTempDirectory("workspace-file-tool-").toAbsolutePath(); + WorkspacePathGuard pathGuard = new WorkspacePathGuard(root); + WorkspaceQuotaGuard quotaGuard = new WorkspaceQuotaGuard( + pathGuard, new WorkspaceQuotaLimits(total, single, count, read), WorkspaceQuotaHook.noop()); + return new Fixture(root, pathGuard, quotaGuard); + } + + private ToolResultBlock call(AgentTool tool, Map input) { + return tool.callAsync(ToolCallParam.builder().input(input).build()).block(); + } + + private String text(ToolResultBlock result) { + return ((TextBlock) result.getOutput().get(0)).getText(); + } + + /** + * 测试工具夹具。 + * + * @param root 工作区根 + * @param pathGuard 路径保护器 + * @param quotaGuard 配额保护器 + */ + private record Fixture(Path root, WorkspacePathGuard pathGuard, WorkspaceQuotaGuard quotaGuard) { + } +} diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuardTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuardTest.java new file mode 100644 index 0000000..5d848b4 --- /dev/null +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/WorkspacePathGuardTest.java @@ -0,0 +1,121 @@ +package com.easyagents.agent.runtime.tool.operate; + +import com.easyagents.agent.runtime.AgentRuntimeException; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * 测试工作区路径边界。 + */ +public class WorkspacePathGuardTest { + + @Test + public void shouldRejectAbsoluteTraversalAndTildePaths() throws IOException { + WorkspacePathGuard guard = guard(); + + assertRejectedWithout(() -> guard.resolveForWrite("/etc/passwd"), "/etc/passwd"); + assertRejected(() -> guard.resolveForWrite("../escape.txt")); + assertRejected(() -> guard.resolveForWrite("~/secret.txt")); + assertRejected(() -> guard.resolveForWrite("C:\\Windows\\system.ini")); + } + + @Test + public void shouldRejectSymbolicLinkEscape() throws IOException { + Path root = Files.createTempDirectory("workspace-path-"); + Path outside = Files.createTempDirectory("workspace-outside-"); + try { + Files.createSymbolicLink(root.resolve("link"), outside); + } catch (UnsupportedOperationException error) { + Assume.assumeNoException(error); + } + WorkspacePathGuard guard = new WorkspacePathGuard(root.toAbsolutePath()); + + assertRejected(() -> guard.resolveForWrite("link/secret.txt")); + } + + @Test + public void shouldRejectHardLinkedFileOnUnix() throws IOException { + Assume.assumeTrue(FileSystems.getDefault().supportedFileAttributeViews().contains("unix")); + Path root = Files.createTempDirectory("workspace-hardlink-"); + Files.writeString(root.resolve("source.txt"), "secret"); + Files.createLink(root.resolve("alias.txt"), root.resolve("source.txt")); + WorkspacePathGuard guard = new WorkspacePathGuard(root.toAbsolutePath()); + + assertRejected(() -> guard.resolveExistingFile("alias.txt")); + } + + @Test + public void shouldRejectSymlinkReplacementBetweenResolveAndAtomicCommit() throws IOException { + Path root = Files.createTempDirectory("workspace-replacement-"); + Path outside = Files.createTempFile("workspace-replacement-outside-", ".txt"); + Files.writeString(outside, "outside"); + WorkspacePathGuard guard = new WorkspacePathGuard(root.toAbsolutePath()); + Path target = guard.resolveForWrite("target.txt"); + try { + Files.createSymbolicLink(target, outside); + } catch (UnsupportedOperationException error) { + Assume.assumeNoException(error); + } + + assertRejected(() -> WorkspaceTextFiles.atomicWrite( + guard, target, "changed".getBytes(StandardCharsets.UTF_8))); + + Assert.assertEquals("outside", Files.readString(outside)); + } + + @Test + public void shouldRejectHardLinkReplacementBetweenResolveAndAtomicCommitOnUnix() throws IOException { + Assume.assumeTrue(FileSystems.getDefault().supportedFileAttributeViews().contains("unix")); + Path root = Files.createTempDirectory("workspace-hardlink-replacement-"); + Path outside = Files.createTempFile("workspace-hardlink-outside-", ".txt"); + Files.writeString(outside, "outside"); + WorkspacePathGuard guard = new WorkspacePathGuard(root.toAbsolutePath()); + Path target = guard.resolveForWrite("target.txt"); + Files.createLink(target, outside); + + assertRejected(() -> WorkspaceTextFiles.atomicWrite( + guard, target, "changed".getBytes(StandardCharsets.UTF_8))); + + Assert.assertEquals("outside", Files.readString(outside)); + } + + @Test + public void shouldDisplayOnlyRelativePath() throws IOException { + Path root = Files.createTempDirectory("workspace-display-"); + WorkspacePathGuard guard = new WorkspacePathGuard(root.toAbsolutePath()); + Path target = guard.resolveForWrite("output/report.txt"); + + Assert.assertEquals(".", guard.display(root.toRealPath())); + Assert.assertEquals("output/report.txt", guard.display(target)); + Assert.assertFalse(guard.display(target).contains(root.toString())); + } + + private WorkspacePathGuard guard() throws IOException { + return new WorkspacePathGuard(Files.createTempDirectory("workspace-path-").toAbsolutePath()); + } + + private void assertRejected(Runnable action) { + try { + action.run(); + Assert.fail("Expected AgentRuntimeException"); + } catch (AgentRuntimeException expected) { + Assert.assertFalse(expected.getMessage().contains("/Users/")); + } + } + + private void assertRejectedWithout(Runnable action, String forbiddenText) { + try { + action.run(); + Assert.fail("Expected AgentRuntimeException"); + } catch (AgentRuntimeException expected) { + Assert.assertFalse(expected.getMessage().contains(forbiddenText)); + } + } +} From c8be1631245ecd3f09d89c92ae5a435fb707d4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Wed, 19 Aug 2026 22:38:01 +0800 Subject: [PATCH 31/33] =?UTF-8?q?feat:=20=E6=89=A9=E5=B1=95=E6=A0=87?= =?UTF-8?q?=E5=87=86=20Skill=20=E5=8C=85=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持单层父目录包装和多 Skill ZIP 解码 - 安全忽略 macOS 元数据并保持路径校验 --- .../skill/codec/ZipSkillPackageCodec.java | 89 +++++++++++++++---- .../com/easyagents/skill/model/Skill.java | 4 +- .../skill/model/SkillPackageLayout.java | 4 +- .../com/easyagents/skill/util/SkillPaths.java | 28 +++++- .../skill/codec/ZipSkillPackageCodecTest.java | 80 ++++++++++++++++- .../easyagents/skill/util/SkillPathsTest.java | 19 ++++ 6 files changed, 197 insertions(+), 27 deletions(-) diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java index 8f14d13..33bc0f3 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/codec/ZipSkillPackageCodec.java @@ -251,9 +251,10 @@ public class ZipSkillPackageCodec implements AutoCloseable { throw packageError("UNSUPPORTED_ZIP_ENTRY", rawPath, "Encrypted or unsupported ZIP entries are not allowed."); } + boolean ignoredSystemPath = SkillPaths.isIgnoredSystemPath(rawPath); String pathForValidation = entry.isDirectory() ? stripDirectorySuffix(rawPath) : rawPath; - String path = normalizeArchivePath(pathForValidation, limits); - if (entry.isDirectory() || SkillPaths.isIgnoredSystemPath(rawPath)) { + String path = normalizeArchivePath(pathForValidation, limits, ignoredSystemPath); + if (entry.isDirectory() || ignoredSystemPath) { continue; } if (!exactPaths.add(path)) { @@ -322,7 +323,7 @@ public class ZipSkillPackageCodec implements AutoCloseable { ArchiveFile skillFile = group.files.stream() .filter(file -> SkillPaths.SKILL_FILE.equals(file.relativePath)) .findFirst() - .orElseThrow(() -> packageError("SKILL_FILE_REQUIRED", group.root, + .orElseThrow(() -> packageError("SKILL_FILE_REQUIRED", group.archiveRoot, "Skill directory must contain exactly one SKILL.md.")); String skillContent = readStrictText(zipFile, skillFile, limits.getMaxTextFileBytes()); @@ -331,7 +332,7 @@ public class ZipSkillPackageCodec implements AutoCloseable { document = SkillFrontmatter.parseDocument(skillContent, limits); } catch (SkillValidationException e) { throw validationPackageError(e, - layout == SkillPackageLayout.ROOT_SKILL ? null : group.root); + layout == SkillPackageLayout.ROOT_SKILL ? null : group.packageRoot); } String name = scalar(document.getFrontmatter().get("name")); @@ -346,7 +347,7 @@ public class ZipSkillPackageCodec implements AutoCloseable { resources.sort(Comparator.comparing(SkillResource::getPath)); Skill skill = new Skill(); - skill.setPackageRoot(layout == SkillPackageLayout.ROOT_SKILL ? name : group.root); + skill.setPackageRoot(layout == SkillPackageLayout.ROOT_SKILL ? name : group.packageRoot); skill.setDocument(document); skill.setResources(resources); return skill; @@ -412,19 +413,44 @@ public class ZipSkillPackageCodec implements AutoCloseable { .map(file -> file.withRelativePath(file.fullPath)) .toList(); return new ArchiveLayout(SkillPackageLayout.ROOT_SKILL, - List.of(new ArchiveGroup(null, relativeFiles))); + List.of(new ArchiveGroup(null, null, relativeFiles))); + } + + List skillRoots = files.stream() + .map(file -> file.fullPath) + .filter(path -> path.endsWith("/" + SkillPaths.SKILL_FILE)) + .map(path -> path.substring(0, path.length() - SkillPaths.SKILL_FILE.length() - 1)) + .distinct() + .sorted() + .toList(); + if (skillRoots.isEmpty()) { + throw packageError("SKILL_FILE_REQUIRED", null, + "Wrapped Skill ZIP must contain at least one Skill directory."); + } + + String commonParent = parentPath(skillRoots.get(0)); + boolean supportedParent = commonParent.isEmpty() || commonParent.indexOf('/') < 0; + boolean sameParent = skillRoots.stream().allMatch(root -> parentPath(root).equals(commonParent)); + if (!supportedParent || !sameParent) { + throw packageError("MIXED_PACKAGE_LAYOUT", null, + "Skill directories must be direct ZIP roots or share one optional parent directory."); } Map> grouped = new LinkedHashMap<>(); + for (String root : skillRoots) { + grouped.put(root, new ArrayList<>()); + } for (ArchiveFile file : files) { - int separator = file.fullPath.indexOf('/'); - if (separator < 1 || separator == file.fullPath.length() - 1) { + String owner = skillRoots.stream() + .filter(root -> file.fullPath.startsWith(root + "/")) + .findFirst() + .orElse(null); + if (owner == null) { throw packageError("UNOWNED_ROOT_FILE", file.fullPath, - "Wrapped Skill ZIP root can contain only Skill directories."); + "Wrapped Skill ZIP can contain files only inside Skill directories."); } - String root = file.fullPath.substring(0, separator); - String relativePath = file.fullPath.substring(separator + 1); - grouped.computeIfAbsent(root, ignored -> new ArrayList<>()) + String relativePath = file.fullPath.substring(owner.length() + 1); + grouped.get(owner) .add(file.withRelativePath(relativePath)); } List groups = new ArrayList<>(); @@ -434,16 +460,38 @@ public class ZipSkillPackageCodec implements AutoCloseable { .count(); if (skillFileCount != 1) { throw packageError("SKILL_FILE_REQUIRED", entry.getKey(), - "Each top-level Skill directory must contain exactly one SKILL.md."); + "Each Skill directory must contain exactly one SKILL.md."); } - groups.add(new ArchiveGroup(entry.getKey(), entry.getValue())); + groups.add(new ArchiveGroup(entry.getKey(), fileName(entry.getKey()), entry.getValue())); } - groups.sort(Comparator.comparing(group -> group.root)); + groups.sort(Comparator.comparing(group -> group.archiveRoot)); SkillPackageLayout layout = groups.size() == 1 ? SkillPackageLayout.SINGLE_DIRECTORY : SkillPackageLayout.MULTI_DIRECTORY; return new ArchiveLayout(layout, groups); } + /** + * 获取归一化归档路径的父目录。 + * + * @param path 归一化归档路径 + * @return 父目录,顶层路径返回空字符串 + */ + private static String parentPath(String path) { + int separator = path.lastIndexOf('/'); + return separator < 0 ? "" : path.substring(0, separator); + } + + /** + * 获取归一化归档路径的末级目录名。 + * + * @param path 归一化归档路径 + * @return 末级目录名 + */ + private static String fileName(String path) { + int separator = path.lastIndexOf('/'); + return separator < 0 ? path : path.substring(separator + 1); + } + /** * 执行 Codec 不可绕过的标准安全校验。 * @@ -1136,10 +1184,13 @@ public class ZipSkillPackageCodec implements AutoCloseable { return total; } - private static String normalizeArchivePath(String rawPath, SkillPackageLimits limits) { + private static String normalizeArchivePath(String rawPath, SkillPackageLimits limits, + boolean ignoredSystemPath) { String normalized; try { - normalized = SkillPaths.normalize(rawPath); + normalized = ignoredSystemPath + ? SkillPaths.normalizeIgnoredSystemPath(rawPath) + : SkillPaths.normalize(rawPath); } catch (SkillValidationException e) { throw packageError("UNSAFE_ENTRY_PATH", rawPath, e.getMessage()); } @@ -1196,7 +1247,7 @@ public class ZipSkillPackageCodec implements AutoCloseable { throw packageError("PATH_LENGTH_LIMIT", path, "Skill path exceeds " + limits.getMaxPathLength() + " characters."); } - if (SkillPaths.depth(path) > limits.getMaxPathDepth()) { + if (path.split("/", -1).length > limits.getMaxPathDepth()) { throw packageError("PATH_DEPTH_LIMIT", path, "Skill path exceeds depth " + limits.getMaxPathDepth() + "."); } @@ -1391,7 +1442,7 @@ public class ZipSkillPackageCodec implements AutoCloseable { } } - private record ArchiveGroup(String root, List files) { + private record ArchiveGroup(String archiveRoot, String packageRoot, List files) { } private record ArchiveLayout(SkillPackageLayout layout, List groups) { diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java index 94145a3..a7696c1 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/Skill.java @@ -17,7 +17,7 @@ public class Skill implements Serializable { private List resources = new ArrayList<>(); /** - * 获取导入包中的顶层目录名;该值不是仓储 ID。 + * 获取 Skill 自身的逻辑目录名;导入包允许在其外再包装一层父目录。 * * @return 包目录名 */ @@ -26,7 +26,7 @@ public class Skill implements Serializable { } /** - * 设置导入包中的顶层目录名。 + * 设置 Skill 自身的逻辑目录名。 * * @param packageRoot 包目录名 */ diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java index 07f58a1..0e8090f 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/model/SkillPackageLayout.java @@ -8,9 +8,9 @@ public enum SkillPackageLayout { /** 根目录直接包含 SKILL.md 的单 Skill 包。 */ ROOT_SKILL, - /** 一个顶层目录包装的单 Skill 包。 */ + /** 一个 Skill 目录组成的单 Skill 包,可再由单层父目录包装。 */ SINGLE_DIRECTORY, - /** 多个顶层 Skill 目录组成的批量包。 */ + /** 多个 Skill 目录组成的批量包,可共享一个单层父目录。 */ MULTI_DIRECTORY } diff --git a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java index 6fe92e4..74cd1f7 100644 --- a/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java +++ b/easy-agents-skill/src/main/java/com/easyagents/skill/util/SkillPaths.java @@ -45,6 +45,32 @@ public final class SkillPaths { * @return 规范化后的路径 */ public static String normalize(String path) { + return normalize(path, false); + } + + /** + * 规范化已识别的系统元数据路径,允许其中保留隐藏路径段供导入器安全忽略。 + * + * @param path 原始系统元数据路径 + * @return 规范化后的系统元数据路径 + * @throws SkillValidationException 路径不属于可忽略系统元数据或包含不安全路径段时抛出 + */ + public static String normalizeIgnoredSystemPath(String path) { + if (!isIgnoredSystemPath(path)) { + throw new SkillValidationException("Skill path is not ignored system metadata: " + path); + } + return normalize(path, true); + } + + /** + * 规范化逻辑路径并按用途控制隐藏路径段。 + * + * @param path 原始路径 + * @param allowHiddenSegments 是否允许隐藏路径段 + * @return 规范化后的路径 + * @throws SkillValidationException 路径非法时抛出 + */ + private static String normalize(String path, boolean allowHiddenSegments) { if (path == null) { throw new SkillValidationException("Skill path is required."); } @@ -63,7 +89,7 @@ public final class SkillPaths { if (segment.isEmpty() || ".".equals(segment) || "..".equals(segment)) { throw new SkillValidationException("Unsafe skill path is not allowed: " + path); } - if (segment.startsWith(".")) { + if (!allowHiddenSegments && segment.startsWith(".")) { throw new SkillValidationException("Hidden skill path is not allowed: " + path); } if (!segment.equals(segment.strip())) { diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java index ee4beaa..48d6f25 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/codec/ZipSkillPackageCodecTest.java @@ -110,6 +110,33 @@ public class ZipSkillPackageCodecTest { Assert.assertEquals("root-skill", result.getSkillPackage().getSkills().get(0).getPackageRoot()); } + /** + * macOS Finder 生成的 AppleDouble 与目录元数据不会影响根目录或单目录 Skill 导入。 + */ + @Test + public void ignoreMacOsMetadataForSingleSkillPackages() { + SkillPackageReadResult rootResult = decode(new ZipSkillPackageCodec(), zip(files( + "SKILL.md", utf8(skillMd("root-skill")), + "references/a.md", utf8("# A"), + "__MACOSX/._SKILL.md", utf8("metadata"), + "__MACOSX/references/._a.md", utf8("metadata"), + ".DS_Store", utf8("metadata") + ))); + Assert.assertEquals(SkillPackageLayout.ROOT_SKILL, rootResult.getSkillPackage().getLayout()); + Assert.assertEquals(1, rootResult.getSkillPackage().getSkills().get(0).getResources().size()); + + SkillPackageReadResult wrappedResult = decode(new ZipSkillPackageCodec(), zip(files( + "docx/SKILL.md", utf8(skillMd("docx")), + "docx/references/a.md", utf8("# A"), + "__MACOSX/docx/._SKILL.md", utf8("metadata"), + "__MACOSX/docx/references/._a.md", utf8("metadata"), + "docx/.DS_Store", utf8("metadata") + ))); + Assert.assertEquals(SkillPackageLayout.SINGLE_DIRECTORY, + wrappedResult.getSkillPackage().getLayout()); + Assert.assertEquals(1, wrappedResult.getSkillPackage().getSkills().get(0).getResources().size()); + } + /** * 多目录包按稳定目录顺序导入。 */ @@ -133,6 +160,50 @@ public class ZipSkillPackageCodecTest { .getSkillPackage().getSkills().stream().map(Skill::getName).toList()); } + /** + * 多个标准 Skill 目录被同一个父目录包装后仍按独立 Skill 解码。 + */ + @Test + public void decodeMultipleSkillsWithParentDirectory() { + SkillPackageReadResult result = decode(new ZipSkillPackageCodec(), zip(files( + "skill-bundle/skill-b/SKILL.md", utf8(skillMd("skill-b")), + "skill-bundle/skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-bundle/skill-a/references/a.md", utf8("# A"), + "__MACOSX/skill-bundle/skill-a/._SKILL.md", utf8("metadata"), + "skill-bundle/.DS_Store", utf8("metadata") + ))); + + Assert.assertEquals(SkillPackageLayout.MULTI_DIRECTORY, result.getSkillPackage().getLayout()); + Assert.assertEquals(List.of("skill-a", "skill-b"), result.getSkillPackage().getSkills().stream() + .map(Skill::getName).toList()); + Assert.assertEquals(List.of("skill-a", "skill-b"), result.getSkillPackage().getSkills().stream() + .map(Skill::getPackageRoot).toList()); + Assert.assertEquals(1, result.getSkillPackage().getSkills().get(0).getResources().size()); + } + + /** + * 父目录包装布局中的散落文件不归属于任何 Skill 时拒绝。 + */ + @Test + public void rejectUnownedFileInParentDirectory() { + assertPackageCode("UNOWNED_ROOT_FILE", zip(files( + "skill-bundle/skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-bundle/skill-b/SKILL.md", utf8(skillMd("skill-b")), + "skill-bundle/README.md", utf8("orphan") + )), SkillPackageReadOptions.defaults()); + } + + /** + * Skill 目录来自不同层级或不同父目录时拒绝,避免产生含糊归属。 + */ + @Test + public void rejectMixedSkillDirectoryParents() { + assertPackageCode("MIXED_PACKAGE_LAYOUT", zip(files( + "skill-a/SKILL.md", utf8(skillMd("skill-a")), + "skill-bundle/skill-b/SKILL.md", utf8(skillMd("skill-b")) + )), SkillPackageReadOptions.defaults()); + } + /** * 嵌套 frontmatter 与未知字段在 Codec 中保留。 */ @@ -268,7 +339,10 @@ public class ZipSkillPackageCodecTest { */ @Test public void rejectUnixSymlink() { - assertPackageCode("SYMLINK_ENTRY", symlinkZip(), SkillPackageReadOptions.defaults()); + assertPackageCode("SYMLINK_ENTRY", symlinkZip("skill-a/assets/link"), + SkillPackageReadOptions.defaults()); + assertPackageCode("SYMLINK_ENTRY", symlinkZip("__MACOSX/._link"), + SkillPackageReadOptions.defaults()); } /** @@ -1121,7 +1195,7 @@ public class ZipSkillPackageCodecTest { return truncated; } - private static byte[] symlinkZip() { + private static byte[] symlinkZip(String path) { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ZipArchiveOutputStream output = new ZipArchiveOutputStream(bytes)) { @@ -1131,7 +1205,7 @@ public class ZipSkillPackageCodecTest { output.write(utf8(skillMd("skill-a"))); output.closeArchiveEntry(); - ZipArchiveEntry symlink = new ZipArchiveEntry("skill-a/assets/link"); + ZipArchiveEntry symlink = new ZipArchiveEntry(path); symlink.setUnixMode(UnixStat.LINK_FLAG | 0777); output.putArchiveEntry(symlink); output.write(utf8("target")); diff --git a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java index f2039d7..b85472b 100644 --- a/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java +++ b/easy-agents-skill/src/test/java/com/easyagents/skill/util/SkillPathsTest.java @@ -32,4 +32,23 @@ public class SkillPathsTest { public void normalizeUnicodeToNfc() { Assert.assertEquals("references/\u00e9.md", SkillPaths.normalize("references/e\u0301.md")); } + + /** + * 已识别的 macOS 系统元数据允许隐藏文件名,但仍执行路径规范化。 + */ + @Test + public void normalizeIgnoredMacOsMetadata() { + Assert.assertEquals("__MACOSX/._docx-js.md", + SkillPaths.normalizeIgnoredSystemPath("__MACOSX/._docx-js.md")); + Assert.assertEquals("docx/.DS_Store", + SkillPaths.normalizeIgnoredSystemPath("docx/.DS_Store")); + } + + /** + * 普通隐藏文件不能借用系统元数据规范化入口。 + */ + @Test(expected = SkillValidationException.class) + public void rejectNonSystemHiddenPathThroughIgnoredNormalizer() { + SkillPaths.normalizeIgnoredSystemPath("docx/.env"); + } } From 870c2cc583608492b680d52feb822b38bf3d18d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 20 Aug 2026 11:23:55 +0800 Subject: [PATCH 32/33] =?UTF-8?q?fix:=20=E7=BB=9F=E4=B8=80=20Shell=20?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E5=BC=80=E5=85=B3=E8=AF=AD=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 关闭 Shell 审批时移除命令级审批策略与强制审批元数据 - 补充适配器与运行时回归测试 --- .../tool/operate/AgentOperateToolAdapter.java | 11 +++++++---- .../agentscope/AgentScopeStatefulRuntimeTest.java | 12 ++++++------ .../tool/operate/AgentOperateToolAdapterTest.java | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java index dc33402..516f231 100644 --- a/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java +++ b/easy-agents-agent-runtime/src/main/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapter.java @@ -119,7 +119,10 @@ public class AgentOperateToolAdapter { toolkit.registerAgentTool(shellTool); AgentToolSpec shellToolSpec = toolSpec( spec, EXECUTE_SHELL_COMMAND_TOOL, "Execute shell command.", true); - shellToolSpec.setApprovalPolicy(shellTool::approvalEvaluation); + if (shellToolSpec.isApprovalRequired()) { + // 命令级审批策略服从 Agent 的 Shell 审批开关;关闭后仅保留安全校验。 + shellToolSpec.setApprovalPolicy(shellTool::approvalEvaluation); + } toolSpecs.add(shellToolSpec); } default -> throw new AgentRuntimeException("Unsupported agent operate tool type: " + type); @@ -157,7 +160,7 @@ public class AgentOperateToolAdapter { toolSpec.setVisibility(AgentToolVisibility.VISIBLE); toolSpec.setApprovalRequired(approvalRequired); toolSpec.setApprovalRequest(approvalRequest(operateSpec, approvalRequired)); - toolSpec.setMetadata(metadata(operateSpec)); + toolSpec.setMetadata(metadata(operateSpec, approvalRequired)); return toolSpec; } @@ -173,11 +176,11 @@ public class AgentOperateToolAdapter { return defaultRequest; } - private Map metadata(AgentOperateToolSpec spec) { + private Map metadata(AgentOperateToolSpec spec, boolean approvalRequired) { Map metadata = new LinkedHashMap<>(); metadata.put("operateTool", true); metadata.put("operateToolType", spec.getType().name()); - if (spec.getType() == AgentOperateToolType.SHELL) { + if (spec.getType() == AgentOperateToolType.SHELL && approvalRequired) { metadata.put("forceApprovalCommands", List.of("rm")); metadata.put("forceApprovalCommandArgument", "command"); } diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java index 1ee90f9..95886fe 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/agentscope/AgentScopeStatefulRuntimeTest.java @@ -282,13 +282,13 @@ public class AgentScopeStatefulRuntimeTest { } @Test - public void shouldForceApprovalForRemoveWhenShellApprovalIsDisabled() { + public void shouldBypassRemoveApprovalWhenShellApprovalIsDisabled() { AgentInitRequest request = initRequest(); AgentOperateToolSpec shell = operateToolSpec(AgentOperateToolType.SHELL); shell.setApprovalRequired(false); request.getAgentDefinition().setOperateToolSpecs(List.of(shell)); AgentScopeReActRuntime runtime = runtimeWithModel(List.of(ChatResponse.builder() - .id("forced-remove-message") + .id("remove-message") .content(List.of(ToolUseBlock.builder() .id("call-remove") .name(AgentOperateToolAdapter.EXECUTE_SHELL_COMMAND_TOOL) @@ -303,11 +303,11 @@ public class AgentScopeStatefulRuntimeTest { .block(Duration.ofSeconds(5)); Assert.assertNotNull(events); - Assert.assertTrue(events.stream() - .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED)); - Assert.assertTrue(events.stream() - .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED)); Assert.assertFalse(events.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_APPROVAL_REQUIRED)); + Assert.assertFalse(events.stream() + .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.SUSPENDED)); + Assert.assertTrue(events.stream() .anyMatch(event -> event.getEventType() == AgentRuntimeEventType.TOOL_RESULT)); } diff --git a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java index 537d507..e4c31ee 100644 --- a/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java +++ b/easy-agents-agent-runtime/src/test/java/com/easyagents/agent/runtime/tool/operate/AgentOperateToolAdapterTest.java @@ -68,6 +68,21 @@ public class AgentOperateToolAdapterTest { Assert.assertFalse(toolSpecs.get(0).getMetadata().containsKey("baseDir")); } + @Test + public void shouldDisableAllShellApprovalPoliciesWithAgentSwitch() { + Toolkit toolkit = new Toolkit(); + AgentOperateToolSpec spec = spec(AgentOperateToolType.SHELL); + spec.setApprovalRequired(false); + + List toolSpecs = adapter.register(List.of(spec), toolkit); + + Assert.assertEquals(1, toolSpecs.size()); + Assert.assertFalse(toolSpecs.get(0).isApprovalRequired()); + Assert.assertNull(toolSpecs.get(0).getApprovalPolicy()); + Assert.assertFalse(toolSpecs.get(0).getMetadata().containsKey("forceApprovalCommands")); + Assert.assertFalse(toolSpecs.get(0).getMetadata().containsKey("forceApprovalCommandArgument")); + } + @Test public void shouldRegisterPatchWithDefaultHitlDisabled() { Toolkit toolkit = new Toolkit(); From 0af5147c1932f815848a569725913df504edc159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 20 Aug 2026 11:34:00 +0800 Subject: [PATCH 33/33] =?UTF-8?q?release:=20=E5=8F=91=E5=B8=83v1.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 69a6bbb..ad4f9b1 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ - 1.1.0-RC + 1.1.0 17 1.3.0 UTF-8