feat: 支持 Skill 仓库扫描与批量导入
- 统一支持本地单包、多包和 Git 候选最多 50 项导入 - 固定提交并限制 SSRF、DNS、并发、频率、超时和仓库资源 - 增加导入对话框、接口契约和安全回归测试
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.net.Inet4Address;
|
||||
import java.net.Inet6Address;
|
||||
import java.net.InetAddress;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Git 仓库远程地址访问策略，负责协议、凭证、端口与 SSRF 边界校验。
|
||||
*/
|
||||
@Component
|
||||
public class GitRepositoryAccessPolicy {
|
||||
|
||||
private final SkillGitImportProperties properties;
|
||||
private final HostResolver hostResolver;
|
||||
|
||||
/**
|
||||
* 创建 Git 地址访问策略。
|
||||
*
|
||||
* @param properties Git 导入配置
|
||||
*/
|
||||
public GitRepositoryAccessPolicy(SkillGitImportProperties properties) {
|
||||
this(properties, InetAddress::getAllByName);
|
||||
}
|
||||
|
||||
GitRepositoryAccessPolicy(SkillGitImportProperties properties, HostResolver hostResolver) {
|
||||
this.properties = properties;
|
||||
this.hostResolver = hostResolver;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并规范化用户输入的 Git 仓库地址。
|
||||
*
|
||||
* @param repositoryUrl 用户输入地址
|
||||
* @return 规范化 HTTPS 地址
|
||||
* @throws BusinessException 地址不符合安全边界
|
||||
*/
|
||||
public String normalizeRepositoryUrl(String repositoryUrl) {
|
||||
if (repositoryUrl == null || repositoryUrl.isBlank()) {
|
||||
throw new BusinessException("请输入 Git 仓库地址");
|
||||
}
|
||||
URI uri;
|
||||
try {
|
||||
uri = new URI(repositoryUrl.trim()).normalize();
|
||||
} catch (URISyntaxException exception) {
|
||||
throw new BusinessException("Git 仓库地址格式不正确");
|
||||
}
|
||||
validateUri(uri, true);
|
||||
String path = uri.getRawPath();
|
||||
if (path == null || path.isBlank() || "/".equals(path)) {
|
||||
throw new BusinessException("Git 仓库地址缺少仓库路径");
|
||||
}
|
||||
String normalizedPath = path.length() > 1 && path.endsWith("/")
|
||||
? path.substring(0, path.length() - 1) : path;
|
||||
try {
|
||||
return new URI("https", null, uri.getHost().toLowerCase(Locale.ROOT),
|
||||
uri.getPort(), normalizedPath, null, null).toASCIIString();
|
||||
} catch (URISyntaxException exception) {
|
||||
throw new BusinessException("Git 仓库地址格式不正确");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验 JGit 实际发起的连接地址，包括重定向后的服务地址。
|
||||
*
|
||||
* @param url 连接地址
|
||||
* @throws java.io.IOException 地址被访问策略拒绝
|
||||
*/
|
||||
ResolvedConnection resolveConnection(URL url) throws IOException {
|
||||
try {
|
||||
URI uri = url.toURI();
|
||||
InetAddress[] addresses = validateUri(uri, false);
|
||||
return new ResolvedConnection(uri.getHost().toLowerCase(Locale.ROOT), addresses);
|
||||
} catch (URISyntaxException | BusinessException exception) {
|
||||
throw new IOException(exception.getMessage(), exception);
|
||||
}
|
||||
}
|
||||
|
||||
private InetAddress[] validateUri(URI uri, boolean repositoryInput) {
|
||||
if (!"https".equalsIgnoreCase(uri.getScheme())) {
|
||||
throw new BusinessException("Git 仓库仅支持 HTTPS 地址");
|
||||
}
|
||||
if (uri.getUserInfo() != null) {
|
||||
throw new BusinessException("Git 仓库地址不能包含用户名或密码");
|
||||
}
|
||||
if (repositoryInput && (uri.getQuery() != null || uri.getFragment() != null)) {
|
||||
throw new BusinessException("Git 仓库地址不能包含查询参数或片段");
|
||||
}
|
||||
String host = uri.getHost();
|
||||
if (host == null || host.isBlank()) {
|
||||
throw new BusinessException("Git 仓库地址缺少有效主机");
|
||||
}
|
||||
int port = uri.getPort() < 0 ? 443 : uri.getPort();
|
||||
if (!properties.getAllowedPorts().contains(port)) {
|
||||
throw new BusinessException("Git 仓库端口不在允许范围内");
|
||||
}
|
||||
String normalizedHost = host.toLowerCase(Locale.ROOT);
|
||||
InetAddress[] addresses;
|
||||
try {
|
||||
addresses = hostResolver.resolve(normalizedHost);
|
||||
} catch (UnknownHostException exception) {
|
||||
throw new BusinessException("无法解析 Git 仓库主机");
|
||||
}
|
||||
if (addresses.length == 0) {
|
||||
throw new BusinessException("无法解析 Git 仓库主机");
|
||||
}
|
||||
if (!properties.getTrustedPrivateHosts().contains(normalizedHost)
|
||||
&& Arrays.stream(addresses).anyMatch(this::isBlockedAddress)) {
|
||||
throw new BusinessException("Git 仓库地址指向受限网络");
|
||||
}
|
||||
return addresses.clone();
|
||||
}
|
||||
|
||||
private boolean isBlockedAddress(InetAddress address) {
|
||||
if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress()
|
||||
|| address.isSiteLocalAddress() || address.isMulticastAddress()) {
|
||||
return true;
|
||||
}
|
||||
byte[] bytes = address.getAddress();
|
||||
if (address instanceof Inet4Address && bytes.length == 4) {
|
||||
int first = Byte.toUnsignedInt(bytes[0]);
|
||||
int second = Byte.toUnsignedInt(bytes[1]);
|
||||
int third = Byte.toUnsignedInt(bytes[2]);
|
||||
return first == 0 || first == 10 || first == 127 || first >= 224
|
||||
|| (first == 100 && second >= 64 && second <= 127)
|
||||
|| (first == 169 && second == 254)
|
||||
|| (first == 172 && second >= 16 && second <= 31)
|
||||
|| (first == 192 && second == 0 && (third == 0 || third == 2))
|
||||
|| (first == 192 && second == 88 && third == 99)
|
||||
|| (first == 192 && second == 168)
|
||||
|| (first == 198 && (second == 18 || second == 19))
|
||||
|| (first == 198 && second == 51 && third == 100)
|
||||
|| (first == 203 && second == 0 && third == 113);
|
||||
}
|
||||
if (address instanceof Inet6Address && bytes.length == 16) {
|
||||
int first = Byte.toUnsignedInt(bytes[0]);
|
||||
int second = Byte.toUnsignedInt(bytes[1]);
|
||||
return (first & 0xFE) == 0xFC || (first == 0xFE && (second & 0xC0) == 0x80)
|
||||
|| (first == 0x20 && second == 0x01 && Byte.toUnsignedInt(bytes[2]) == 0x0D
|
||||
&& Byte.toUnsignedInt(bytes[3]) == 0xB8);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
interface HostResolver {
|
||||
|
||||
/**
|
||||
* 解析目标主机的全部地址。
|
||||
*
|
||||
* @param host 规范化主机名
|
||||
* @return 解析地址
|
||||
* @throws UnknownHostException 主机无法解析
|
||||
*/
|
||||
InetAddress[] resolve(String host) throws UnknownHostException;
|
||||
}
|
||||
|
||||
record ResolvedConnection(String host, InetAddress[] addresses) {
|
||||
|
||||
ResolvedConnection {
|
||||
addresses = addresses.clone();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取用于实际连接的固定地址。
|
||||
*
|
||||
* @return 地址副本
|
||||
*/
|
||||
@Override
|
||||
public InetAddress[] addresses() {
|
||||
return addresses.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import com.easyagents.skill.exception.SkillValidationException;
|
||||
import com.easyagents.skill.factory.SkillFactory;
|
||||
import com.easyagents.skill.model.Skill;
|
||||
import com.easyagents.skill.model.SkillPackageLimits;
|
||||
import com.easyagents.skill.util.SkillHashes;
|
||||
import com.easyagents.skill.util.SkillPaths;
|
||||
import com.easyagents.skill.util.SkillResources;
|
||||
import com.easyagents.skill.validation.SkillValidationIssue;
|
||||
import com.easyagents.skill.validation.SkillValidationSeverity;
|
||||
import com.easyagents.skill.validation.defaults.DefaultSkillValidator;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
import org.eclipse.jgit.lib.FileMode;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.ObjectLoader;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.revwalk.RevCommit;
|
||||
import org.eclipse.jgit.revwalk.RevWalk;
|
||||
import org.eclipse.jgit.treewalk.TreeWalk;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.CharacterCodingException;
|
||||
import java.nio.charset.CodingErrorAction;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
/**
|
||||
* 固定 Git Tree 中的标准 Skill 候选发现、校验与标准包组装器。
|
||||
*/
|
||||
@Component
|
||||
public class SkillGitCandidateScanner {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SkillGitCandidateScanner.class);
|
||||
private static final int LFS_POINTER_MAX_BYTES = 1_024;
|
||||
private static final byte[] LFS_HEADER = "version https://git-lfs.github.com/spec/v1"
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
private final SkillGitImportProperties properties;
|
||||
private final SkillPackageLimits packageLimits = SkillPackageLimits.defaults();
|
||||
private final DefaultSkillValidator validator = new DefaultSkillValidator();
|
||||
|
||||
/**
|
||||
* 创建候选扫描器。
|
||||
*
|
||||
* @param properties Git 导入配置
|
||||
*/
|
||||
public SkillGitCandidateScanner(SkillGitImportProperties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 扫描快照固定提交中的全部标准 Skill 候选。
|
||||
*
|
||||
* @param snapshot 固定提交快照
|
||||
* @return 候选及其资源归属
|
||||
* @throws BusinessException 仓库结构不可读取或超过安全限额
|
||||
*/
|
||||
public SkillGitRepositoryScan scan(SkillGitRepositoryReader.Snapshot snapshot) {
|
||||
Repository repository = snapshot.repository();
|
||||
try (RevWalk revWalk = new RevWalk(repository)) {
|
||||
RevCommit commit = revWalk.parseCommit(ObjectId.fromString(snapshot.commitSha()));
|
||||
List<RawEntry> entries = readEntries(repository, commit);
|
||||
Set<String> roots = discoverCandidateRoots(entries);
|
||||
if (roots.size() > properties.getMaxCandidates()) {
|
||||
throw new BusinessException(413, 4133, "Git 仓库中的 Skill 候选数量超过限制");
|
||||
}
|
||||
Map<String, List<RawEntry>> ownedEntries = assignEntries(entries, roots);
|
||||
List<SkillGitScanResult.Candidate> candidates = new ArrayList<>(roots.size());
|
||||
Map<String, List<SkillGitRepositoryScan.RepositoryFile>> filesByCandidate = new LinkedHashMap<>();
|
||||
roots.stream().sorted().forEach(root -> {
|
||||
CandidateBuild build = buildCandidate(repository, snapshot.commitSha(), root,
|
||||
ownedEntries.getOrDefault(root, List.of()));
|
||||
candidates.add(build.candidate());
|
||||
filesByCandidate.put(build.candidate().candidateId(), build.files());
|
||||
});
|
||||
return new SkillGitRepositoryScan(candidates, filesByCandidate);
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException(500, 500, "读取 Git 仓库文件树失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将一个可导入候选组装为包含单个标准 Skill 的 ZIP。
|
||||
*
|
||||
* @param snapshot 固定提交快照
|
||||
* @param scan 同一快照的扫描数据
|
||||
* @param candidateId 候选 ID
|
||||
* @return 待交给标准包预检的临时 ZIP
|
||||
* @throws BusinessException 候选不存在、不可导入或文件读取失败
|
||||
*/
|
||||
public Path createStandardZip(SkillGitRepositoryReader.Snapshot snapshot,
|
||||
SkillGitRepositoryScan scan,
|
||||
String candidateId) {
|
||||
SkillGitScanResult.Candidate candidate = scan.candidates().stream()
|
||||
.filter(item -> item.candidateId().equals(candidateId))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new BusinessException("Git Skill 候选不存在，请重新扫描"));
|
||||
if (!candidate.importable()) {
|
||||
throw new BusinessException("Git Skill 候选存在校验问题，不能导入：" + candidate.path());
|
||||
}
|
||||
List<SkillGitRepositoryScan.RepositoryFile> files = scan.filesByCandidate().get(candidateId);
|
||||
if (files == null || files.stream().noneMatch(file -> SkillPaths.SKILL_FILE.equals(file.relativePath()))) {
|
||||
throw new BusinessException("Git Skill 候选缺少 SKILL.md，请重新扫描");
|
||||
}
|
||||
Path archive;
|
||||
try {
|
||||
archive = Files.createTempFile("easyflow-git-skill-", ".zip");
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException(500, 500, "创建 Git Skill 临时包失败", exception);
|
||||
}
|
||||
try (ZipOutputStream output = new ZipOutputStream(Files.newOutputStream(archive))) {
|
||||
for (SkillGitRepositoryScan.RepositoryFile file : files.stream()
|
||||
.sorted(Comparator.comparing(SkillGitRepositoryScan.RepositoryFile::relativePath)).toList()) {
|
||||
FileMode mode = FileMode.fromBits(file.fileModeBits());
|
||||
if (!isRegular(mode)) {
|
||||
throw new BusinessException("Git Skill 包含不支持的文件对象：" + file.relativePath());
|
||||
}
|
||||
ZipEntry entry = new ZipEntry(candidate.name() + "/" + file.relativePath());
|
||||
entry.setTime(0L);
|
||||
output.putNextEntry(entry);
|
||||
snapshot.repository().open(file.objectId(), Constants.OBJ_BLOB).copyTo(output);
|
||||
output.closeEntry();
|
||||
}
|
||||
return archive;
|
||||
} catch (IOException | RuntimeException exception) {
|
||||
deleteArchive(archive);
|
||||
if (exception instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException(500, 500, "组装 Git Skill 标准包失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除扫描器创建的临时标准包。
|
||||
*
|
||||
* @param archive 临时 ZIP
|
||||
*/
|
||||
public void deleteArchive(Path archive) {
|
||||
if (archive == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(archive);
|
||||
} catch (IOException exception) {
|
||||
// 临时包删除失败不能覆盖已生成的导入预览；记录完整路径供运维清理。
|
||||
LOG.warn("清理 Git Skill 临时包失败，path={}", archive, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private List<RawEntry> readEntries(Repository repository, RevCommit commit) throws IOException {
|
||||
List<RawEntry> entries = new ArrayList<>();
|
||||
try (TreeWalk treeWalk = new TreeWalk(repository)) {
|
||||
treeWalk.addTree(commit.getTree());
|
||||
treeWalk.setRecursive(true);
|
||||
while (treeWalk.next()) {
|
||||
if (entries.size() >= properties.getMaxRepositoryFiles()) {
|
||||
throw new BusinessException(413, 4134, "Git 仓库文件数量超过限制");
|
||||
}
|
||||
FileMode mode = treeWalk.getFileMode(0);
|
||||
ObjectId objectId = treeWalk.getObjectId(0).copy();
|
||||
long size = isRegular(mode) || FileMode.SYMLINK.equals(mode)
|
||||
? repository.open(objectId, Constants.OBJ_BLOB).getSize() : 0L;
|
||||
entries.add(new RawEntry(treeWalk.getPathString(), objectId, mode, size));
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private Set<String> discoverCandidateRoots(List<RawEntry> entries) {
|
||||
Set<String> roots = new LinkedHashSet<>();
|
||||
for (RawEntry entry : entries) {
|
||||
if (isRegular(entry.mode()) && SkillPaths.SKILL_FILE.equals(fileName(entry.path()))) {
|
||||
roots.add(parentPath(entry.path()));
|
||||
}
|
||||
}
|
||||
return roots;
|
||||
}
|
||||
|
||||
private Map<String, List<RawEntry>> assignEntries(List<RawEntry> entries, Set<String> roots) {
|
||||
Map<String, List<RawEntry>> result = new HashMap<>();
|
||||
roots.forEach(root -> result.put(root, new ArrayList<>()));
|
||||
for (RawEntry entry : entries) {
|
||||
String owner = nearestCandidateRoot(entry.path(), roots);
|
||||
if (owner != null) {
|
||||
result.get(owner).add(entry);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String nearestCandidateRoot(String filePath, Set<String> roots) {
|
||||
String parent = parentPath(filePath);
|
||||
while (parent != null) {
|
||||
if (roots.contains(parent)) {
|
||||
return parent;
|
||||
}
|
||||
parent = parent.isEmpty() ? null : parentPath(parent);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private CandidateBuild buildCandidate(Repository repository, String commitSha, String root,
|
||||
List<RawEntry> ownedEntries) {
|
||||
RawEntry skillFile = ownedEntries.stream()
|
||||
.filter(entry -> isRegular(entry.mode()))
|
||||
.filter(entry -> SkillPaths.SKILL_FILE.equals(relativePath(root, entry.path())))
|
||||
.findFirst().orElseThrow(() -> new BusinessException("Git Skill 候选缺少 SKILL.md"));
|
||||
List<SkillGitScanResult.Issue> issues = new ArrayList<>();
|
||||
String fallbackName = root.isEmpty() ? "root-skill" : fileName(root);
|
||||
String name = fallbackName;
|
||||
String description = "";
|
||||
String skillContent = null;
|
||||
if (skillFile.size() > packageLimits.getMaxTextFileBytes()) {
|
||||
issues.add(issue("TEXT_FILE_SIZE_LIMIT", "SKILL.md 超过允许大小", SkillPaths.SKILL_FILE));
|
||||
} else {
|
||||
try {
|
||||
skillContent = decodeUtf8(loadBytes(repository, skillFile));
|
||||
Skill skill = SkillFactory.create(skillContent);
|
||||
skill.setPackageRoot(root.isEmpty() ? skill.getName() : fileName(root));
|
||||
name = skill.getName();
|
||||
description = skill.getDescription();
|
||||
validator.validateReport(skill, packageLimits).getIssues().stream()
|
||||
.filter(item -> item.getSeverity() == SkillValidationSeverity.ERROR)
|
||||
.map(this::toIssue)
|
||||
.forEach(issues::add);
|
||||
} catch (SkillValidationException exception) {
|
||||
issues.add(issue(exception.getCode() == null ? "INVALID_SKILL" : exception.getCode(),
|
||||
exception.getMessage(), SkillPaths.SKILL_FILE));
|
||||
} catch (CharacterCodingException exception) {
|
||||
issues.add(issue("INVALID_UTF8", "SKILL.md 不是有效的 UTF-8 文本", SkillPaths.SKILL_FILE));
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException(500, 500, "读取 Git SKILL.md 失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
List<SkillGitRepositoryScan.RepositoryFile> files = new ArrayList<>(ownedEntries.size());
|
||||
Set<String> collisionKeys = new HashSet<>();
|
||||
collisionKeys.add(SkillPaths.collisionKey(SkillPaths.SKILL_FILE));
|
||||
long totalBytes = 0L;
|
||||
int resourceCount = 0;
|
||||
for (RawEntry entry : ownedEntries) {
|
||||
String relative = relativePath(root, entry.path());
|
||||
totalBytes = addSize(totalBytes, entry.size());
|
||||
files.add(new SkillGitRepositoryScan.RepositoryFile(entry.path(), relative,
|
||||
entry.objectId(), entry.size(), entry.mode().getBits()));
|
||||
if (SkillPaths.SKILL_FILE.equals(relative)) {
|
||||
continue;
|
||||
}
|
||||
resourceCount++;
|
||||
validateResource(repository, entry, relative, collisionKeys, issues);
|
||||
}
|
||||
if (resourceCount + 1 > packageLimits.getMaxEntryCount()) {
|
||||
issues.add(issue("ENTRY_COUNT_LIMIT", "Skill 文件数量超过允许上限", null));
|
||||
}
|
||||
if (totalBytes > packageLimits.getMaxTotalUncompressedBytes()) {
|
||||
issues.add(issue("TOTAL_SIZE_LIMIT", "Skill 资源总大小超过允许上限", null));
|
||||
}
|
||||
String candidateId = SkillHashes.sha256Hex((commitSha + "\n" + root + "\n"
|
||||
+ skillFile.objectId().name()).getBytes(StandardCharsets.UTF_8));
|
||||
SkillGitScanResult.CandidateStatus status = issues.isEmpty()
|
||||
? SkillGitScanResult.CandidateStatus.IMPORTABLE
|
||||
: SkillGitScanResult.CandidateStatus.NEEDS_ATTENTION;
|
||||
SkillGitScanResult.Candidate candidate = new SkillGitScanResult.Candidate(
|
||||
candidateId, root.isEmpty() ? "/" : root, name, description, status,
|
||||
resourceCount, totalBytes, issues);
|
||||
return new CandidateBuild(candidate, files);
|
||||
}
|
||||
|
||||
private void validateResource(Repository repository, RawEntry entry, String relative,
|
||||
Set<String> collisionKeys, List<SkillGitScanResult.Issue> issues) {
|
||||
if (FileMode.SYMLINK.equals(entry.mode())) {
|
||||
issues.add(issue("GIT_SYMLINK_UNSUPPORTED", "暂不支持符号链接", relative));
|
||||
return;
|
||||
}
|
||||
if (FileMode.GITLINK.equals(entry.mode())) {
|
||||
issues.add(issue("GIT_SUBMODULE_UNSUPPORTED", "暂不支持 Git 子模块", relative));
|
||||
return;
|
||||
}
|
||||
if (!isRegular(entry.mode())) {
|
||||
issues.add(issue("GIT_OBJECT_UNSUPPORTED", "不支持该 Git 文件对象类型", relative));
|
||||
return;
|
||||
}
|
||||
String normalized;
|
||||
try {
|
||||
normalized = SkillPaths.normalize(relative);
|
||||
} catch (SkillValidationException exception) {
|
||||
issues.add(issue("UNSAFE_RESOURCE_PATH", "资源路径不符合 Skill 规范", relative));
|
||||
return;
|
||||
}
|
||||
String collisionKey = SkillPaths.collisionKey(normalized);
|
||||
if (!collisionKeys.add(collisionKey)) {
|
||||
issues.add(issue("DUPLICATE_RESOURCE_PATH", "资源路径存在大小写或 Unicode 冲突", relative));
|
||||
}
|
||||
if (normalized.length() > packageLimits.getMaxPathLength()
|
||||
|| SkillPaths.depth(normalized) + 1 > packageLimits.getMaxPathDepth()) {
|
||||
issues.add(issue("RESOURCE_PATH_LIMIT", "资源路径长度或深度超过限制", relative));
|
||||
}
|
||||
String mediaType = mediaType(relative);
|
||||
boolean text = SkillResources.isText(relative, mediaType);
|
||||
long fileLimit = text ? packageLimits.getMaxTextFileBytes() : packageLimits.getMaxBinaryFileBytes();
|
||||
if (entry.size() > fileLimit) {
|
||||
issues.add(issue(text ? "TEXT_FILE_SIZE_LIMIT" : "BINARY_FILE_SIZE_LIMIT",
|
||||
"资源文件超过允许大小", relative));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
byte[] bytes = null;
|
||||
if (text) {
|
||||
bytes = loadBytes(repository, entry);
|
||||
decodeUtf8(bytes);
|
||||
}
|
||||
if (entry.size() <= LFS_POINTER_MAX_BYTES
|
||||
&& isLfsPointer(bytes == null ? loadBytes(repository, entry) : bytes)) {
|
||||
issues.add(issue("GIT_LFS_UNRESOLVED", "Git LFS 文件尚未包含真实内容", relative));
|
||||
}
|
||||
} catch (CharacterCodingException exception) {
|
||||
issues.add(issue("INVALID_UTF8", "文本资源不是有效的 UTF-8 内容", relative));
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException(500, 500, "读取 Git Skill 资源失败：" + relative, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] loadBytes(Repository repository, RawEntry entry) throws IOException {
|
||||
ObjectLoader loader = repository.open(entry.objectId(), Constants.OBJ_BLOB);
|
||||
long maximum = Math.max(packageLimits.getMaxBinaryFileBytes(), packageLimits.getMaxTextFileBytes());
|
||||
if (loader.getSize() > maximum) {
|
||||
throw new IOException("Git Blob exceeds the configured single-file limit");
|
||||
}
|
||||
return loader.getBytes((int) Math.min(Integer.MAX_VALUE, maximum));
|
||||
}
|
||||
|
||||
private String decodeUtf8(byte[] bytes) throws CharacterCodingException {
|
||||
return StandardCharsets.UTF_8.newDecoder()
|
||||
.onMalformedInput(CodingErrorAction.REPORT)
|
||||
.onUnmappableCharacter(CodingErrorAction.REPORT)
|
||||
.decode(ByteBuffer.wrap(bytes)).toString();
|
||||
}
|
||||
|
||||
private boolean isLfsPointer(byte[] bytes) {
|
||||
if (bytes.length < LFS_HEADER.length) {
|
||||
return false;
|
||||
}
|
||||
for (int index = 0; index < LFS_HEADER.length; index++) {
|
||||
if (bytes[index] != LFS_HEADER[index]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private SkillGitScanResult.Issue toIssue(SkillValidationIssue source) {
|
||||
return issue(source.getCode(), source.getMessage(), source.getPath());
|
||||
}
|
||||
|
||||
private SkillGitScanResult.Issue issue(String code, String message, String path) {
|
||||
return new SkillGitScanResult.Issue(code, message, path);
|
||||
}
|
||||
|
||||
private String mediaType(String path) {
|
||||
String guessed = URLConnection.guessContentTypeFromName(path);
|
||||
return guessed == null ? "application/octet-stream" : guessed;
|
||||
}
|
||||
|
||||
private long addSize(long total, long size) {
|
||||
try {
|
||||
return Math.addExact(total, Math.max(0L, size));
|
||||
} catch (ArithmeticException exception) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isRegular(FileMode mode) {
|
||||
return FileMode.REGULAR_FILE.equals(mode) || FileMode.EXECUTABLE_FILE.equals(mode);
|
||||
}
|
||||
|
||||
private String relativePath(String root, String path) {
|
||||
return root.isEmpty() ? path : path.substring(root.length() + 1);
|
||||
}
|
||||
|
||||
private String parentPath(String path) {
|
||||
int index = path.lastIndexOf('/');
|
||||
return index < 0 ? "" : path.substring(0, index);
|
||||
}
|
||||
|
||||
private String fileName(String path) {
|
||||
int index = path.lastIndexOf('/');
|
||||
return index < 0 ? path : path.substring(index + 1);
|
||||
}
|
||||
|
||||
private record RawEntry(String path, ObjectId objectId, FileMode mode, long size) {
|
||||
}
|
||||
|
||||
private record CandidateBuild(SkillGitScanResult.Candidate candidate,
|
||||
List<SkillGitRepositoryScan.RepositoryFile> files) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Skill Git 仓库导入的网络与资源边界配置。
|
||||
*/
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "easyflow.skill.git-import")
|
||||
public class SkillGitImportProperties {
|
||||
|
||||
/** Git 网络操作超时时间。 */
|
||||
private Duration timeout = Duration.ofSeconds(45);
|
||||
|
||||
/** 单次 Git 远程操作总超时时间。 */
|
||||
private Duration operationTimeout = Duration.ofSeconds(90);
|
||||
|
||||
/** 单实例允许同时执行的 Git 远程操作数量。 */
|
||||
private int maxConcurrentOperations = 4;
|
||||
|
||||
/** 同一账号两次仓库扫描之间的最小间隔。 */
|
||||
private Duration scanMinimumInterval = Duration.ofSeconds(2);
|
||||
|
||||
/** 扫描令牌有效期。 */
|
||||
private Duration scanTtl = Duration.ofMinutes(30);
|
||||
|
||||
/** 临时裸仓库最大磁盘占用。 */
|
||||
private long maxRepositoryBytes = 256L * 1024 * 1024;
|
||||
|
||||
/** 单仓库最大文件对象数量。 */
|
||||
private int maxRepositoryFiles = 50_000;
|
||||
|
||||
/** 单仓库最大 Skill 候选数量。 */
|
||||
private int maxCandidates = 2_000;
|
||||
|
||||
/** 允许访问的 HTTPS 端口。 */
|
||||
private Set<Integer> allowedPorts = new LinkedHashSet<>(Set.of(443));
|
||||
|
||||
/** 允许解析到私网地址的可信 Git 主机。 */
|
||||
private Set<String> trustedPrivateHosts = new LinkedHashSet<>();
|
||||
|
||||
/**
|
||||
* 获取 Git 网络操作超时时间。
|
||||
*
|
||||
* @return 超时时间
|
||||
*/
|
||||
public Duration getTimeout() {
|
||||
return timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 Git 网络操作超时时间。
|
||||
*
|
||||
* @param timeout 超时时间
|
||||
*/
|
||||
public void setTimeout(Duration timeout) {
|
||||
this.timeout = timeout == null ? Duration.ofSeconds(45) : timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取单次 Git 远程操作总超时时间。
|
||||
*
|
||||
* @return 总超时时间
|
||||
*/
|
||||
public Duration getOperationTimeout() {
|
||||
return operationTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置单次 Git 远程操作总超时时间。
|
||||
*
|
||||
* @param operationTimeout 总超时时间
|
||||
*/
|
||||
public void setOperationTimeout(Duration operationTimeout) {
|
||||
this.operationTimeout = positive(operationTimeout, "operationTimeout");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取单实例 Git 远程操作并发上限。
|
||||
*
|
||||
* @return 并发上限
|
||||
*/
|
||||
public int getMaxConcurrentOperations() {
|
||||
return maxConcurrentOperations;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置单实例 Git 远程操作并发上限。
|
||||
*
|
||||
* @param maxConcurrentOperations 并发上限
|
||||
*/
|
||||
public void setMaxConcurrentOperations(int maxConcurrentOperations) {
|
||||
this.maxConcurrentOperations = positive(maxConcurrentOperations, "maxConcurrentOperations");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取同一账号两次仓库扫描的最小间隔。
|
||||
*
|
||||
* @return 最小间隔
|
||||
*/
|
||||
public Duration getScanMinimumInterval() {
|
||||
return scanMinimumInterval;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置同一账号两次仓库扫描的最小间隔。
|
||||
*
|
||||
* @param scanMinimumInterval 最小间隔
|
||||
*/
|
||||
public void setScanMinimumInterval(Duration scanMinimumInterval) {
|
||||
this.scanMinimumInterval = positive(scanMinimumInterval, "scanMinimumInterval");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取扫描令牌有效期。
|
||||
*
|
||||
* @return 有效期
|
||||
*/
|
||||
public Duration getScanTtl() {
|
||||
return scanTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置扫描令牌有效期。
|
||||
*
|
||||
* @param scanTtl 有效期
|
||||
*/
|
||||
public void setScanTtl(Duration scanTtl) {
|
||||
this.scanTtl = scanTtl == null ? Duration.ofMinutes(30) : scanTtl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取临时仓库大小上限。
|
||||
*
|
||||
* @return 字节数
|
||||
*/
|
||||
public long getMaxRepositoryBytes() {
|
||||
return maxRepositoryBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置临时仓库大小上限。
|
||||
*
|
||||
* @param maxRepositoryBytes 字节数
|
||||
*/
|
||||
public void setMaxRepositoryBytes(long maxRepositoryBytes) {
|
||||
this.maxRepositoryBytes = positive(maxRepositoryBytes, "maxRepositoryBytes");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取仓库文件对象数量上限。
|
||||
*
|
||||
* @return 文件对象数量
|
||||
*/
|
||||
public int getMaxRepositoryFiles() {
|
||||
return maxRepositoryFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置仓库文件对象数量上限。
|
||||
*
|
||||
* @param maxRepositoryFiles 文件对象数量
|
||||
*/
|
||||
public void setMaxRepositoryFiles(int maxRepositoryFiles) {
|
||||
this.maxRepositoryFiles = positive(maxRepositoryFiles, "maxRepositoryFiles");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 Skill 候选数量上限。
|
||||
*
|
||||
* @return 候选数量
|
||||
*/
|
||||
public int getMaxCandidates() {
|
||||
return maxCandidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置 Skill 候选数量上限。
|
||||
*
|
||||
* @param maxCandidates 候选数量
|
||||
*/
|
||||
public void setMaxCandidates(int maxCandidates) {
|
||||
this.maxCandidates = positive(maxCandidates, "maxCandidates");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取允许访问的 HTTPS 端口。
|
||||
*
|
||||
* @return 端口集合
|
||||
*/
|
||||
public Set<Integer> getAllowedPorts() {
|
||||
return allowedPorts;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置允许访问的 HTTPS 端口。
|
||||
*
|
||||
* @param allowedPorts 端口集合
|
||||
*/
|
||||
public void setAllowedPorts(Set<Integer> allowedPorts) {
|
||||
LinkedHashSet<Integer> normalized = new LinkedHashSet<>();
|
||||
if (allowedPorts != null) {
|
||||
allowedPorts.stream().filter(port -> port != null && port > 0 && port <= 65_535)
|
||||
.forEach(normalized::add);
|
||||
}
|
||||
this.allowedPorts = normalized.isEmpty() ? new LinkedHashSet<>(Set.of(443)) : normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取可信私网 Git 主机。
|
||||
*
|
||||
* @return 小写主机集合
|
||||
*/
|
||||
public Set<String> getTrustedPrivateHosts() {
|
||||
return trustedPrivateHosts;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置可信私网 Git 主机。
|
||||
*
|
||||
* @param trustedPrivateHosts 主机集合
|
||||
*/
|
||||
public void setTrustedPrivateHosts(Set<String> trustedPrivateHosts) {
|
||||
LinkedHashSet<String> normalized = new LinkedHashSet<>();
|
||||
if (trustedPrivateHosts != null) {
|
||||
trustedPrivateHosts.stream().filter(host -> host != null && !host.isBlank())
|
||||
.map(host -> host.trim().toLowerCase(java.util.Locale.ROOT))
|
||||
.forEach(normalized::add);
|
||||
}
|
||||
this.trustedPrivateHosts = normalized;
|
||||
}
|
||||
|
||||
private static int positive(int value, String name) {
|
||||
if (value <= 0) {
|
||||
throw new IllegalArgumentException(name + " must be positive");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static long positive(long value, String name) {
|
||||
if (value <= 0) {
|
||||
throw new IllegalArgumentException(name + " must be positive");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static Duration positive(Duration value, String name) {
|
||||
if (value == null || value.isZero() || value.isNegative()) {
|
||||
throw new IllegalArgumentException(name + " must be positive");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.skill.imports.SkillImportPreview;
|
||||
import tech.easyflow.skill.imports.SkillImportService;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 编排 Git 仓库扫描、固定提交复核与标准 Skill 导入预览。
|
||||
*/
|
||||
@Service
|
||||
public class SkillGitImportService {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SkillGitImportService.class);
|
||||
private static final int MAX_PREPARE_COUNT = SkillImportService.MAX_BATCH_SKILL_COUNT;
|
||||
|
||||
private final GitRepositoryAccessPolicy accessPolicy;
|
||||
private final SkillGitRepositoryReader repositoryReader;
|
||||
private final SkillGitCandidateScanner candidateScanner;
|
||||
private final SkillGitScanStore scanStore;
|
||||
private final SkillGitScanRateLimiter scanRateLimiter;
|
||||
private final SkillImportService skillImportService;
|
||||
|
||||
/**
|
||||
* 创建 Git Skill 导入编排服务。
|
||||
*
|
||||
* @param accessPolicy 仓库地址访问策略
|
||||
* @param repositoryReader Git 仓库读取器
|
||||
* @param candidateScanner Skill 候选扫描器
|
||||
* @param scanStore 短期扫描会话仓库
|
||||
* @param scanRateLimiter 仓库扫描频控器
|
||||
* @param skillImportService 标准 Skill ZIP 导入服务
|
||||
*/
|
||||
public SkillGitImportService(GitRepositoryAccessPolicy accessPolicy,
|
||||
SkillGitRepositoryReader repositoryReader,
|
||||
SkillGitCandidateScanner candidateScanner,
|
||||
SkillGitScanStore scanStore,
|
||||
SkillGitScanRateLimiter scanRateLimiter,
|
||||
SkillImportService skillImportService) {
|
||||
this.accessPolicy = accessPolicy;
|
||||
this.repositoryReader = repositoryReader;
|
||||
this.candidateScanner = candidateScanner;
|
||||
this.scanStore = scanStore;
|
||||
this.scanRateLimiter = scanRateLimiter;
|
||||
this.skillImportService = skillImportService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 扫描任意受支持的 HTTPS Git 仓库并登记短期固定提交结果。
|
||||
*
|
||||
* @param repositoryUrl Git 仓库地址，可省略 .git 后缀
|
||||
* @return 候选扫描结果
|
||||
*/
|
||||
public SkillGitScanResult scan(String repositoryUrl) {
|
||||
scanRateLimiter.check();
|
||||
String normalizedUrl = accessPolicy.normalizeRepositoryUrl(repositoryUrl);
|
||||
try (SkillGitRepositoryReader.Snapshot snapshot = repositoryReader.openDefault(normalizedUrl)) {
|
||||
SkillGitRepositoryScan repositoryScan = candidateScanner.scan(snapshot);
|
||||
SkillGitScanResult.RepositoryInfo repository = new SkillGitScanResult.RepositoryInfo(
|
||||
repositoryName(normalizedUrl), normalizedUrl,
|
||||
Repository.shortenRefName(snapshot.branchRef()), snapshot.commitSha());
|
||||
return scanStore.create(repository, snapshot.branchRef(), repositoryScan.candidates());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 复核扫描时的固定提交，并把选中候选转换为既有标准包导入预览。
|
||||
*
|
||||
* @param scanToken 扫描令牌
|
||||
* @param candidateIds 选中的候选 ID
|
||||
* @return 与现有 ZIP 导入一致的预览列表
|
||||
*/
|
||||
public List<SkillImportPreview> prepare(String scanToken, List<String> candidateIds) {
|
||||
validateCandidateIds(candidateIds);
|
||||
return scanStore.withLockedSession(scanToken,
|
||||
session -> prepareLocked(scanToken, session, List.copyOf(candidateIds)));
|
||||
}
|
||||
|
||||
private List<SkillImportPreview> prepareLocked(String scanToken,
|
||||
SkillGitScanStore.Session session,
|
||||
List<String> candidateIds) {
|
||||
SkillGitScanResult original = session.result();
|
||||
Set<String> allowedIds = new HashSet<>();
|
||||
original.candidates().stream().filter(SkillGitScanResult.Candidate::importable)
|
||||
.map(SkillGitScanResult.Candidate::candidateId).forEach(allowedIds::add);
|
||||
if (!allowedIds.containsAll(candidateIds)) {
|
||||
throw new BusinessException("所选 Git Skill 不存在或当前不可导入，请重新扫描");
|
||||
}
|
||||
|
||||
List<SkillImportPreview> previews = new ArrayList<>(candidateIds.size());
|
||||
try (SkillGitRepositoryReader.Snapshot snapshot = repositoryReader.openCommit(
|
||||
original.repository().url(), session.branchRef(), original.repository().commitSha())) {
|
||||
SkillGitRepositoryScan current = candidateScanner.scan(snapshot);
|
||||
Set<String> currentImportableIds = new HashSet<>();
|
||||
current.candidates().stream().filter(SkillGitScanResult.Candidate::importable)
|
||||
.map(SkillGitScanResult.Candidate::candidateId).forEach(currentImportableIds::add);
|
||||
if (!currentImportableIds.containsAll(candidateIds)) {
|
||||
throw new BusinessException("Git 仓库扫描结果已变化，请重新扫描");
|
||||
}
|
||||
for (String candidateId : candidateIds) {
|
||||
SkillGitScanResult.Candidate candidate = current.candidates().stream()
|
||||
.filter(item -> item.candidateId().equals(candidateId))
|
||||
.findFirst().orElseThrow(() -> new BusinessException("Git Skill 候选不存在，请重新扫描"));
|
||||
Path archive = candidateScanner.createStandardZip(snapshot, current, candidateId);
|
||||
try {
|
||||
previews.add(skillImportService.preview(candidate.name() + ".zip", archive.toFile()));
|
||||
} finally {
|
||||
candidateScanner.deleteArchive(archive);
|
||||
}
|
||||
}
|
||||
scanStore.remove(scanToken);
|
||||
return List.copyOf(previews);
|
||||
} catch (RuntimeException exception) {
|
||||
cancelPreparedPreviews(previews);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private void cancelPreparedPreviews(List<SkillImportPreview> previews) {
|
||||
for (SkillImportPreview preview : previews) {
|
||||
if (preview == null || preview.getImportToken() == null || preview.getImportToken().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
skillImportService.cancel(preview.getImportToken());
|
||||
} catch (RuntimeException cleanupFailure) {
|
||||
LOG.error("回滚 Git Skill 导入预览失败，token={}", preview.getImportToken(), cleanupFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void validateCandidateIds(List<String> candidateIds) {
|
||||
if (candidateIds == null || candidateIds.isEmpty()) {
|
||||
throw new BusinessException("请选择要导入的 Git Skill");
|
||||
}
|
||||
if (candidateIds.size() > MAX_PREPARE_COUNT) {
|
||||
throw new BusinessException("单次最多导入 " + MAX_PREPARE_COUNT + " 个 Git Skill");
|
||||
}
|
||||
if (candidateIds.stream().anyMatch(id -> id == null || !id.matches("^[a-fA-F0-9]{64}$"))) {
|
||||
throw new BusinessException("Git Skill 候选参数格式不正确");
|
||||
}
|
||||
if (new HashSet<>(candidateIds).size() != candidateIds.size()) {
|
||||
throw new BusinessException("不能重复选择同一个 Git Skill");
|
||||
}
|
||||
}
|
||||
|
||||
private String repositoryName(String repositoryUrl) {
|
||||
try {
|
||||
String path = new URI(repositoryUrl).getPath();
|
||||
String name = path.substring(path.lastIndexOf('/') + 1);
|
||||
return name.toLowerCase(java.util.Locale.ROOT).endsWith(".git")
|
||||
? name.substring(0, name.length() - 4) : name;
|
||||
} catch (URISyntaxException | RuntimeException exception) {
|
||||
throw new BusinessException("Git 仓库地址格式不正确");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
/**
|
||||
* 以有界线程池和总超时执行 Git 远程操作。
|
||||
*/
|
||||
@Component
|
||||
public class SkillGitOperationExecutor {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SkillGitOperationExecutor.class);
|
||||
|
||||
private final SkillGitImportProperties properties;
|
||||
private final ThreadPoolExecutor executor;
|
||||
|
||||
/**
|
||||
* 创建 Git 远程操作执行器。
|
||||
*
|
||||
* @param properties Git 导入配置
|
||||
*/
|
||||
public SkillGitOperationExecutor(SkillGitImportProperties properties) {
|
||||
this.properties = properties;
|
||||
int concurrency = properties.getMaxConcurrentOperations();
|
||||
this.executor = new ThreadPoolExecutor(concurrency, concurrency,
|
||||
30L, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory(),
|
||||
new ThreadPoolExecutor.AbortPolicy());
|
||||
this.executor.allowCoreThreadTimeOut(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在并发与总超时边界内执行远程操作。
|
||||
*
|
||||
* @param operation 远程操作
|
||||
* @param <T> 返回值类型
|
||||
* @return 操作结果
|
||||
* @throws BusinessException 操作繁忙、超时或执行失败
|
||||
*/
|
||||
public <T> T execute(Callable<T> operation) {
|
||||
return execute(operation, ignored -> {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 在并发与总超时边界内执行远程操作，并清理超时后才产出的结果。
|
||||
*
|
||||
* @param operation 远程操作
|
||||
* @param abandonedResultCleanup 超时或中断后结果清理器
|
||||
* @param <T> 返回值类型
|
||||
* @return 操作结果
|
||||
* @throws BusinessException 操作繁忙、超时或执行失败
|
||||
*/
|
||||
public <T> T execute(Callable<T> operation, Consumer<T> abandonedResultCleanup) {
|
||||
AtomicBoolean abandoned = new AtomicBoolean();
|
||||
AtomicReference<T> produced = new AtomicReference<>();
|
||||
Future<T> future;
|
||||
try {
|
||||
future = executor.submit(() -> {
|
||||
T result = operation.call();
|
||||
produced.set(result);
|
||||
cleanupIfAbandoned(abandoned, produced, abandonedResultCleanup);
|
||||
return result;
|
||||
});
|
||||
} catch (java.util.concurrent.RejectedExecutionException exception) {
|
||||
throw new BusinessException(429, 42921, "Git 导入任务繁忙，请稍后重试", exception);
|
||||
}
|
||||
try {
|
||||
T result = future.get(properties.getOperationTimeout().toMillis(), TimeUnit.MILLISECONDS);
|
||||
produced.compareAndSet(result, null);
|
||||
return result;
|
||||
} catch (TimeoutException exception) {
|
||||
abandoned.set(true);
|
||||
future.cancel(true);
|
||||
cleanupIfAbandoned(abandoned, produced, abandonedResultCleanup);
|
||||
throw new BusinessException(504, 50421, "Git 仓库读取超时，请稍后重试", exception);
|
||||
} catch (InterruptedException exception) {
|
||||
abandoned.set(true);
|
||||
future.cancel(true);
|
||||
cleanupIfAbandoned(abandoned, produced, abandonedResultCleanup);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException(503, 50321, "Git 仓库读取被中断，请稍后重试", exception);
|
||||
} catch (ExecutionException exception) {
|
||||
Throwable cause = exception.getCause();
|
||||
if (cause instanceof RuntimeException runtimeException) {
|
||||
throw runtimeException;
|
||||
}
|
||||
if (cause instanceof Error error) {
|
||||
throw error;
|
||||
}
|
||||
throw new BusinessException(502, 5021, "Git 仓库读取失败，请稍后重试", cause);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> void cleanupIfAbandoned(AtomicBoolean abandoned,
|
||||
AtomicReference<T> produced,
|
||||
Consumer<T> abandonedResultCleanup) {
|
||||
if (!abandoned.get()) {
|
||||
return;
|
||||
}
|
||||
T result = produced.getAndSet(null);
|
||||
if (result != null) {
|
||||
try {
|
||||
abandonedResultCleanup.accept(result);
|
||||
} catch (RuntimeException exception) {
|
||||
LOG.error("清理已放弃的 Git 远程操作结果失败", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止执行器并中断尚未结束的远程操作。
|
||||
*/
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
private ThreadFactory threadFactory() {
|
||||
AtomicInteger sequence = new AtomicInteger();
|
||||
return task -> {
|
||||
Thread thread = new Thread(task, "easyflow-skill-git-" + sequence.incrementAndGet());
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.config.Registry;
|
||||
import org.apache.http.config.RegistryBuilder;
|
||||
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
||||
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.apache.http.ssl.SSLContexts;
|
||||
import org.eclipse.jgit.api.Git;
|
||||
import org.eclipse.jgit.api.LsRemoteCommand;
|
||||
import org.eclipse.jgit.api.TransportConfigCallback;
|
||||
import org.eclipse.jgit.api.errors.GitAPIException;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.Ref;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.transport.RefSpec;
|
||||
import org.eclipse.jgit.transport.TagOpt;
|
||||
import org.eclipse.jgit.transport.Transport;
|
||||
import org.eclipse.jgit.transport.TransportHttp;
|
||||
import org.eclipse.jgit.transport.http.HttpConnection;
|
||||
import org.eclipse.jgit.transport.http.HttpConnectionFactory;
|
||||
import org.eclipse.jgit.transport.http.HttpConnectionFactory2;
|
||||
import org.eclipse.jgit.transport.http.apache.HttpClientConnection;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.ProtocolException;
|
||||
import java.net.Proxy;
|
||||
import java.net.URL;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.KeyManagementException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import javax.net.ssl.HostnameVerifier;
|
||||
import javax.net.ssl.KeyManager;
|
||||
import javax.net.ssl.TrustManager;
|
||||
|
||||
/**
|
||||
* 基于 Eclipse JGit 的受控只读仓库读取器。
|
||||
*/
|
||||
@Component
|
||||
public class SkillGitRepositoryReader {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SkillGitRepositoryReader.class);
|
||||
private static final String PINNED_REF = "refs/easyflow/skill-import";
|
||||
|
||||
private final GitRepositoryAccessPolicy accessPolicy;
|
||||
private final SkillGitImportProperties properties;
|
||||
private final SkillGitOperationExecutor operationExecutor;
|
||||
|
||||
/**
|
||||
* 创建 Git 仓库读取器。
|
||||
*
|
||||
* @param accessPolicy 地址访问策略
|
||||
* @param properties Git 导入配置
|
||||
* @param operationExecutor Git 远程操作执行器
|
||||
*/
|
||||
public SkillGitRepositoryReader(GitRepositoryAccessPolicy accessPolicy,
|
||||
SkillGitImportProperties properties,
|
||||
SkillGitOperationExecutor operationExecutor) {
|
||||
this.accessPolicy = accessPolicy;
|
||||
this.properties = properties;
|
||||
this.operationExecutor = operationExecutor;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取远程默认分支的当前固定提交。
|
||||
*
|
||||
* @param repositoryUrl 规范化仓库地址
|
||||
* @return 临时只读仓库快照
|
||||
* @throws BusinessException 远程仓库不可读取或超过资源限制
|
||||
*/
|
||||
public Snapshot openDefault(String repositoryUrl) {
|
||||
return operationExecutor.execute(() -> openDefaultInternal(repositoryUrl), Snapshot::close);
|
||||
}
|
||||
|
||||
private Snapshot openDefaultInternal(String repositoryUrl) {
|
||||
RemoteHead remoteHead = resolveRemoteHead(repositoryUrl);
|
||||
return cloneBranch(repositoryUrl, remoteHead.branchRef(), remoteHead.commitSha());
|
||||
}
|
||||
|
||||
/**
|
||||
* 重新读取扫描时固定的提交；默认分支已变化时尝试按提交 SHA 获取。
|
||||
*
|
||||
* @param repositoryUrl 规范化仓库地址
|
||||
* @param branchRef 扫描时默认分支完整引用
|
||||
* @param commitSha 扫描时完整提交 SHA
|
||||
* @return 固定提交快照
|
||||
* @throws BusinessException 提交不可读取或仓库超过资源限制
|
||||
*/
|
||||
public Snapshot openCommit(String repositoryUrl, String branchRef, String commitSha) {
|
||||
return operationExecutor.execute(
|
||||
() -> openCommitInternal(repositoryUrl, branchRef, commitSha), Snapshot::close);
|
||||
}
|
||||
|
||||
private Snapshot openCommitInternal(String repositoryUrl, String branchRef, String commitSha) {
|
||||
Snapshot branchSnapshot = cloneBranch(repositoryUrl, branchRef, null);
|
||||
if (commitSha.equals(branchSnapshot.commitSha())) {
|
||||
return branchSnapshot;
|
||||
}
|
||||
try {
|
||||
RefSpec pinned = new RefSpec().setForceUpdate(true)
|
||||
.setSourceDestination(commitSha, PINNED_REF);
|
||||
// Git.wrap 与快照共享 Repository；此处不能关闭包装器，否则会提前关闭后续扫描所需的仓库。
|
||||
Git.wrap(branchSnapshot.repository()).fetch()
|
||||
.setRemote(Constants.DEFAULT_REMOTE_NAME)
|
||||
.setRefSpecs(pinned)
|
||||
.setDepth(1)
|
||||
.setTagOpt(TagOpt.NO_TAGS)
|
||||
.setTimeout(timeoutSeconds())
|
||||
.setTransportConfigCallback(transportCallback())
|
||||
.call();
|
||||
ObjectId pinnedId = branchSnapshot.repository().resolve(PINNED_REF);
|
||||
if (pinnedId == null || !commitSha.equals(pinnedId.name())) {
|
||||
throw new BusinessException("扫描时的 Git 提交已不可读取，请重新扫描");
|
||||
}
|
||||
enforceRepositorySize(branchSnapshot.directory());
|
||||
return branchSnapshot.withCommit(commitSha);
|
||||
} catch (GitAPIException | IOException exception) {
|
||||
branchSnapshot.close();
|
||||
throw remoteFailure("扫描时的 Git 提交已不可读取，请重新扫描", exception);
|
||||
} catch (RuntimeException exception) {
|
||||
branchSnapshot.close();
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private RemoteHead resolveRemoteHead(String repositoryUrl) {
|
||||
try {
|
||||
LsRemoteCommand command = Git.lsRemoteRepository()
|
||||
.setRemote(repositoryUrl)
|
||||
.setTimeout(timeoutSeconds())
|
||||
.setTransportConfigCallback(transportCallback());
|
||||
Map<String, Ref> refs = command.callAsMap();
|
||||
Ref head = refs.get(Constants.HEAD);
|
||||
if (head == null || head.getObjectId() == null) {
|
||||
throw new BusinessException("Git 仓库没有可读取的默认分支");
|
||||
}
|
||||
Ref branch = resolveHeadBranch(head, refs.values());
|
||||
if (branch == null) {
|
||||
throw new BusinessException("无法识别 Git 仓库默认分支");
|
||||
}
|
||||
return new RemoteHead(branch.getName(), head.getObjectId().name());
|
||||
} catch (GitAPIException exception) {
|
||||
throw remoteFailure("无法读取 Git 仓库，请检查地址和访问权限", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private Ref resolveHeadBranch(Ref head, Collection<Ref> refs) {
|
||||
if (head.isSymbolic() && head.getTarget() != null
|
||||
&& head.getTarget().getName().startsWith(Constants.R_HEADS)) {
|
||||
return head.getTarget();
|
||||
}
|
||||
return refs.stream()
|
||||
.filter(ref -> ref.getName().startsWith(Constants.R_HEADS))
|
||||
.filter(ref -> head.getObjectId().equals(ref.getObjectId()))
|
||||
.sorted(Comparator.comparingInt(this::branchPriority).thenComparing(Ref::getName))
|
||||
.findFirst().orElse(null);
|
||||
}
|
||||
|
||||
private int branchPriority(Ref ref) {
|
||||
return switch (ref.getName()) {
|
||||
case Constants.R_HEADS + "main" -> 0;
|
||||
case Constants.R_HEADS + Constants.MASTER -> 1;
|
||||
default -> 2;
|
||||
};
|
||||
}
|
||||
|
||||
private Snapshot cloneBranch(String repositoryUrl, String branchRef, String expectedCommit) {
|
||||
Path directory = createTemporaryDirectory();
|
||||
Path gitDirectory = directory.resolve("repository.git");
|
||||
Repository repository = null;
|
||||
try {
|
||||
Git git = Git.cloneRepository()
|
||||
.setURI(repositoryUrl)
|
||||
.setDirectory(gitDirectory.toFile())
|
||||
.setBare(true)
|
||||
.setNoCheckout(true)
|
||||
.setBranchesToClone(java.util.List.of(branchRef))
|
||||
.setBranch(branchRef)
|
||||
.setDepth(1)
|
||||
.setTagOption(TagOpt.NO_TAGS)
|
||||
.setTimeout(timeoutSeconds())
|
||||
.setTransportConfigCallback(transportCallback())
|
||||
.call();
|
||||
repository = git.getRepository();
|
||||
ObjectId head = repository.resolve(Constants.HEAD);
|
||||
if (head == null) {
|
||||
repository.close();
|
||||
deleteDirectory(directory);
|
||||
throw new BusinessException("Git 仓库默认分支没有可读取的提交");
|
||||
}
|
||||
enforceRepositorySize(directory);
|
||||
if (expectedCommit != null && !expectedCommit.equals(head.name())) {
|
||||
repository.close();
|
||||
deleteDirectory(directory);
|
||||
throw new BusinessException("Git 仓库默认分支在扫描期间发生变化，请重新扫描");
|
||||
}
|
||||
return new Snapshot(repository, directory, branchRef, head.name());
|
||||
} catch (GitAPIException | IOException exception) {
|
||||
if (repository != null) {
|
||||
repository.close();
|
||||
}
|
||||
deleteDirectory(directory);
|
||||
throw remoteFailure("无法读取 Git 仓库，请检查地址和访问权限", exception);
|
||||
} catch (RuntimeException exception) {
|
||||
if (repository != null) {
|
||||
repository.close();
|
||||
}
|
||||
deleteDirectory(directory);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private TransportConfigCallback transportCallback() {
|
||||
HttpConnectionFactory factory = new PolicyHttpConnectionFactory(
|
||||
accessPolicy, properties.getMaxRepositoryBytes(), timeoutMillis());
|
||||
int timeout = timeoutSeconds();
|
||||
return transport -> configureTransport(transport, factory, timeout);
|
||||
}
|
||||
|
||||
private void configureTransport(Transport transport, HttpConnectionFactory factory, int timeout) {
|
||||
if (!(transport instanceof TransportHttp httpTransport)) {
|
||||
throw new IllegalArgumentException("Git 仓库仅支持 HTTPS 传输");
|
||||
}
|
||||
transport.setTimeout(timeout);
|
||||
httpTransport.setHttpConnectionFactory(factory);
|
||||
}
|
||||
|
||||
private int timeoutSeconds() {
|
||||
long seconds = Math.max(1L, properties.getTimeout().toSeconds());
|
||||
return (int) Math.min(Integer.MAX_VALUE, seconds);
|
||||
}
|
||||
|
||||
private int timeoutMillis() {
|
||||
long millis = Math.max(1L, properties.getTimeout().toMillis());
|
||||
return (int) Math.min(Integer.MAX_VALUE, millis);
|
||||
}
|
||||
|
||||
private Path createTemporaryDirectory() {
|
||||
try {
|
||||
return Files.createTempDirectory("easyflow-skill-git-");
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException(500, 500, "创建 Git 仓库临时目录失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private void enforceRepositorySize(Path directory) {
|
||||
try (java.util.stream.Stream<Path> paths = Files.walk(directory)) {
|
||||
long total = 0L;
|
||||
java.util.Iterator<Path> iterator = paths.filter(Files::isRegularFile).iterator();
|
||||
while (iterator.hasNext()) {
|
||||
total = Math.addExact(total, Files.size(iterator.next()));
|
||||
if (total > properties.getMaxRepositoryBytes()) {
|
||||
throw new BusinessException(413, 4132, "Git 仓库超过允许的临时存储大小");
|
||||
}
|
||||
}
|
||||
} catch (ArithmeticException exception) {
|
||||
throw new BusinessException(413, 4132, "Git 仓库超过允许的临时存储大小");
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException(500, 500, "检查 Git 仓库临时文件失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private BusinessException remoteFailure(String message, Exception exception) {
|
||||
return new BusinessException(502, 5021, message, exception);
|
||||
}
|
||||
|
||||
private static void deleteDirectory(Path directory) {
|
||||
if (directory == null || !Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
try (java.util.stream.Stream<Path> paths = Files.walk(directory)) {
|
||||
paths.sorted(Comparator.reverseOrder()).forEach(path -> {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException exception) {
|
||||
LOG.warn("清理 Skill Git 临时文件失败，path={}", path, exception);
|
||||
}
|
||||
});
|
||||
} catch (IOException exception) {
|
||||
LOG.warn("遍历 Skill Git 临时目录失败，path={}", directory, exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 单次固定提交的临时仓库快照。
|
||||
*/
|
||||
public static final class Snapshot implements AutoCloseable {
|
||||
|
||||
private final Repository repository;
|
||||
private final Path directory;
|
||||
private final String branchRef;
|
||||
private final String commitSha;
|
||||
|
||||
Snapshot(Repository repository, Path directory, String branchRef, String commitSha) {
|
||||
this.repository = repository;
|
||||
this.directory = directory;
|
||||
this.branchRef = branchRef;
|
||||
this.commitSha = commitSha;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 JGit 仓库。
|
||||
*
|
||||
* @return 只读仓库
|
||||
*/
|
||||
public Repository repository() {
|
||||
return repository;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取临时目录。
|
||||
*
|
||||
* @return 临时目录
|
||||
*/
|
||||
public Path directory() {
|
||||
return directory;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取默认分支完整引用。
|
||||
*
|
||||
* @return 分支引用
|
||||
*/
|
||||
public String branchRef() {
|
||||
return branchRef;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取固定提交 SHA。
|
||||
*
|
||||
* @return 完整提交 SHA
|
||||
*/
|
||||
public String commitSha() {
|
||||
return commitSha;
|
||||
}
|
||||
|
||||
private Snapshot withCommit(String pinnedCommit) {
|
||||
return new Snapshot(repository, directory, branchRef, pinnedCommit);
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭仓库并清理临时目录。
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
repository.close();
|
||||
deleteDirectory(directory);
|
||||
}
|
||||
}
|
||||
|
||||
private record RemoteHead(String branchRef, String commitSha) {
|
||||
}
|
||||
|
||||
private static final class PolicyHttpConnectionFactory implements HttpConnectionFactory2 {
|
||||
|
||||
private final GitRepositoryAccessPolicy accessPolicy;
|
||||
private final long maxResponseBytes;
|
||||
private final AtomicLong responseBytes = new AtomicLong();
|
||||
private final Map<String, InetAddress[]> pinnedAddresses = new ConcurrentHashMap<>();
|
||||
private final CloseableHttpClient httpClient;
|
||||
|
||||
private PolicyHttpConnectionFactory(GitRepositoryAccessPolicy accessPolicy,
|
||||
long maxResponseBytes,
|
||||
int timeoutMillis) {
|
||||
this.accessPolicy = accessPolicy;
|
||||
this.maxResponseBytes = maxResponseBytes;
|
||||
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
|
||||
.register("https", new SSLConnectionSocketFactory(
|
||||
SSLContexts.createSystemDefault(),
|
||||
SSLConnectionSocketFactory.getDefaultHostnameVerifier()))
|
||||
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
||||
.build();
|
||||
PoolingHttpClientConnectionManager connectionManager =
|
||||
new PoolingHttpClientConnectionManager(registry, this::resolvePinnedAddress);
|
||||
connectionManager.setMaxTotal(4);
|
||||
connectionManager.setDefaultMaxPerRoute(2);
|
||||
RequestConfig requestConfig = RequestConfig.custom()
|
||||
.setConnectTimeout(timeoutMillis)
|
||||
.setConnectionRequestTimeout(timeoutMillis)
|
||||
.setSocketTimeout(timeoutMillis)
|
||||
.setRedirectsEnabled(false)
|
||||
.build();
|
||||
this.httpClient = HttpClients.custom()
|
||||
.setConnectionManager(connectionManager)
|
||||
.setDefaultRequestConfig(requestConfig)
|
||||
.disableAutomaticRetries()
|
||||
.disableRedirectHandling()
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpConnection create(URL url) throws IOException {
|
||||
return create(url, Proxy.NO_PROXY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpConnection create(URL url, Proxy proxy) throws IOException {
|
||||
if (proxy != null && !Proxy.NO_PROXY.equals(proxy)) {
|
||||
throw new IOException("Git repository proxy access is disabled");
|
||||
}
|
||||
GitRepositoryAccessPolicy.ResolvedConnection resolved = accessPolicy.resolveConnection(url);
|
||||
pinnedAddresses.putIfAbsent(resolved.host(), resolved.addresses());
|
||||
return limited(new HttpClientConnection(url.toString(), Proxy.NO_PROXY, httpClient));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建一次 JGit HTTP 会话并在结束时释放专用客户端。
|
||||
*
|
||||
* @return Git HTTP 会话
|
||||
*/
|
||||
@Override
|
||||
public GitSession newSession() {
|
||||
return new GitSession() {
|
||||
@Override
|
||||
public HttpConnection configure(HttpConnection connection, boolean sslVerify) throws IOException {
|
||||
if (!sslVerify) {
|
||||
throw new IOException("Git repository TLS verification must remain enabled");
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
try {
|
||||
httpClient.close();
|
||||
} catch (IOException exception) {
|
||||
LOG.warn("关闭 Skill Git HTTP 客户端失败", exception);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private InetAddress[] resolvePinnedAddress(String host) throws UnknownHostException {
|
||||
InetAddress[] addresses = pinnedAddresses.get(host.toLowerCase(Locale.ROOT));
|
||||
if (addresses == null || addresses.length == 0) {
|
||||
throw new UnknownHostException("Git repository host was not approved: " + host);
|
||||
}
|
||||
return addresses.clone();
|
||||
}
|
||||
|
||||
private HttpConnection limited(HttpConnection connection) {
|
||||
return new LimitedHttpConnection(connection, responseBytes, maxResponseBytes);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 对 JGit HTTP 响应流实施单次远程操作的累计字节上限。
|
||||
*/
|
||||
private static final class LimitedHttpConnection implements HttpConnection {
|
||||
|
||||
private final HttpConnection delegate;
|
||||
private final AtomicLong totalBytes;
|
||||
private final long maximumBytes;
|
||||
private InputStream inputStream;
|
||||
|
||||
private LimitedHttpConnection(HttpConnection delegate, AtomicLong totalBytes, long maximumBytes) {
|
||||
this.delegate = delegate;
|
||||
this.totalBytes = totalBytes;
|
||||
this.maximumBytes = maximumBytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getResponseCode() throws IOException {
|
||||
return delegate.getResponseCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL getURL() {
|
||||
return delegate.getURL();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getResponseMessage() throws IOException {
|
||||
return delegate.getResponseMessage();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<String>> getHeaderFields() {
|
||||
return delegate.getHeaderFields();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setRequestProperty(String key, String value) {
|
||||
delegate.setRequestProperty(key, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setRequestMethod(String method) throws ProtocolException {
|
||||
delegate.setRequestMethod(method);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setUseCaches(boolean useCaches) {
|
||||
delegate.setUseCaches(useCaches);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setConnectTimeout(int timeout) {
|
||||
delegate.setConnectTimeout(timeout);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setReadTimeout(int timeout) {
|
||||
delegate.setReadTimeout(timeout);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
return delegate.getContentType();
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getInputStream() throws IOException {
|
||||
if (inputStream == null) {
|
||||
int contentLength = delegate.getContentLength();
|
||||
if (contentLength > 0 && totalBytes.get() + contentLength > maximumBytes) {
|
||||
throw new IOException("Git repository response exceeds the configured size limit");
|
||||
}
|
||||
inputStream = new FilterInputStream(delegate.getInputStream()) {
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
int value = super.read();
|
||||
if (value >= 0) {
|
||||
addBytes(1L);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] buffer, int offset, int length) throws IOException {
|
||||
int count = super.read(buffer, offset, length);
|
||||
if (count > 0) {
|
||||
addBytes(count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
};
|
||||
}
|
||||
return inputStream;
|
||||
}
|
||||
|
||||
private void addBytes(long count) throws IOException {
|
||||
if (totalBytes.addAndGet(count) > maximumBytes) {
|
||||
throw new IOException("Git repository response exceeds the configured size limit");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHeaderField(String name) {
|
||||
return delegate.getHeaderField(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getHeaderFields(String name) {
|
||||
return delegate.getHeaderFields(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getContentLength() {
|
||||
return delegate.getContentLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setInstanceFollowRedirects(boolean followRedirects) {
|
||||
delegate.setInstanceFollowRedirects(followRedirects);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDoOutput(boolean doOutput) {
|
||||
delegate.setDoOutput(doOutput);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setFixedLengthStreamingMode(int contentLength) {
|
||||
delegate.setFixedLengthStreamingMode(contentLength);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OutputStream getOutputStream() throws IOException {
|
||||
return delegate.getOutputStream();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setChunkedStreamingMode(int chunkLength) {
|
||||
delegate.setChunkedStreamingMode(chunkLength);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRequestMethod() {
|
||||
return delegate.getRequestMethod();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean usingProxy() {
|
||||
return delegate.usingProxy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void connect() throws IOException {
|
||||
delegate.connect();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(KeyManager[] keyManagers, TrustManager[] trustManagers,
|
||||
SecureRandom random)
|
||||
throws NoSuchAlgorithmException, KeyManagementException {
|
||||
delegate.configure(keyManagers, trustManagers, random);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setHostnameVerifier(HostnameVerifier verifier)
|
||||
throws NoSuchAlgorithmException, KeyManagementException {
|
||||
delegate.setHostnameVerifier(verifier);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 单次仓库读取期间的候选扫描数据。
|
||||
*
|
||||
* @param candidates 对外候选列表
|
||||
* @param filesByCandidate 候选对应的普通文件对象
|
||||
*/
|
||||
record SkillGitRepositoryScan(
|
||||
List<SkillGitScanResult.Candidate> candidates,
|
||||
Map<String, List<RepositoryFile>> filesByCandidate) {
|
||||
|
||||
/**
|
||||
* 创建不可变扫描数据。
|
||||
*/
|
||||
SkillGitRepositoryScan {
|
||||
candidates = List.copyOf(candidates);
|
||||
filesByCandidate = Map.copyOf(filesByCandidate);
|
||||
}
|
||||
|
||||
/**
|
||||
* 固定提交中的普通文件对象。
|
||||
*
|
||||
* @param path 仓库相对路径
|
||||
* @param relativePath Skill 根目录相对路径
|
||||
* @param objectId Git Blob ID
|
||||
* @param size 原始字节数
|
||||
* @param fileModeBits Git 文件模式
|
||||
*/
|
||||
record RepositoryFile(String path, String relativePath, ObjectId objectId,
|
||||
long size, int fileModeBits) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import com.alicp.jetcache.Cache;
|
||||
import com.alicp.jetcache.CacheInvokeException;
|
||||
import com.alicp.jetcache.CacheResult;
|
||||
import com.alicp.jetcache.CacheResultCode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 对同一租户账号的 Git 仓库扫描实施短周期频控。
|
||||
*/
|
||||
@Component
|
||||
public class SkillGitScanRateLimiter {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SkillGitScanRateLimiter.class);
|
||||
private static final String CACHE_PREFIX = "skill:git-scan:rate:";
|
||||
|
||||
private final Cache<String, Object> defaultCache;
|
||||
private final SkillGitImportProperties properties;
|
||||
|
||||
/**
|
||||
* 创建仓库扫描频控器。
|
||||
*
|
||||
* @param defaultCache 平台默认缓存
|
||||
* @param properties Git 导入配置
|
||||
*/
|
||||
public SkillGitScanRateLimiter(@Qualifier("defaultCache") Cache<String, Object> defaultCache,
|
||||
SkillGitImportProperties properties) {
|
||||
this.defaultCache = defaultCache;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查当前账号是否允许发起新的仓库扫描。
|
||||
*
|
||||
* @throws BusinessException 未登录、请求过于频繁或缓存不可用
|
||||
*/
|
||||
public void check() {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
if (account == null || account.getTenantId() == null || account.getId() == null) {
|
||||
throw new BusinessException(401, 401, "未登录或登录态无效");
|
||||
}
|
||||
check(account);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用已认证账号执行频控检查，供契约测试复用。
|
||||
*
|
||||
* @param account 已认证账号
|
||||
* @throws BusinessException 请求过于频繁或缓存不可用
|
||||
*/
|
||||
void check(LoginAccount account) {
|
||||
long intervalMillis = properties.getScanMinimumInterval().toMillis();
|
||||
String key = CACHE_PREFIX + account.getTenantId() + ":" + account.getId();
|
||||
try {
|
||||
CacheResult result = defaultCache.PUT_IF_ABSENT(
|
||||
key, Boolean.TRUE, intervalMillis, TimeUnit.MILLISECONDS);
|
||||
if (result.getResultCode() == CacheResultCode.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (result.getResultCode() == CacheResultCode.EXISTS) {
|
||||
throw new BusinessException(429, 42922, "Git 仓库扫描过于频繁，请稍后重试");
|
||||
}
|
||||
LOG.error("Git 仓库扫描频控缓存写入失败，code={}, message={}",
|
||||
result.getResultCode(), result.getMessage());
|
||||
throw new BusinessException(503, 50322, "Git 仓库扫描暂不可用，请稍后重试");
|
||||
} catch (CacheInvokeException exception) {
|
||||
LOG.error("Git 仓库扫描频控缓存访问失败", exception);
|
||||
throw new BusinessException(503, 50322, "Git 仓库扫描暂不可用，请稍后重试", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Git 仓库 Skill 候选扫描结果。
|
||||
*
|
||||
* @param scanToken 短期扫描令牌
|
||||
* @param repository 仓库固定提交信息
|
||||
* @param summary 候选数量摘要
|
||||
* @param candidates 候选列表
|
||||
* @param expiresAt 令牌过期时间
|
||||
*/
|
||||
public record SkillGitScanResult(
|
||||
String scanToken,
|
||||
RepositoryInfo repository,
|
||||
Summary summary,
|
||||
List<Candidate> candidates,
|
||||
Date expiresAt) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 创建不可变扫描结果。
|
||||
*/
|
||||
public SkillGitScanResult {
|
||||
candidates = candidates == null ? List.of() : List.copyOf(candidates);
|
||||
expiresAt = expiresAt == null ? null : new Date(expiresAt.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回防御性复制的过期时间。
|
||||
*
|
||||
* @return 过期时间
|
||||
*/
|
||||
@Override
|
||||
public Date expiresAt() {
|
||||
return expiresAt == null ? null : new Date(expiresAt.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* 固定提交的仓库信息。
|
||||
*
|
||||
* @param name 仓库名称
|
||||
* @param url 规范化仓库地址
|
||||
* @param defaultBranch 默认分支
|
||||
* @param commitSha 完整提交 SHA
|
||||
*/
|
||||
public record RepositoryInfo(String name, String url, String defaultBranch,
|
||||
String commitSha) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
|
||||
/**
|
||||
* 候选数量摘要。
|
||||
*
|
||||
* @param discovered 发现数量
|
||||
* @param importable 可导入数量
|
||||
* @param needsAttention 需处理数量
|
||||
*/
|
||||
public record Summary(int discovered, int importable, int needsAttention) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单个 Skill 候选。
|
||||
*
|
||||
* @param candidateId 固定提交内稳定候选 ID
|
||||
* @param path 仓库相对路径
|
||||
* @param name 标准 Skill 名称
|
||||
* @param description 用途描述
|
||||
* @param status 候选状态
|
||||
* @param resourceCount 资源文件数量
|
||||
* @param totalBytes 候选总字节数
|
||||
* @param issues 校验问题
|
||||
*/
|
||||
public record Candidate(String candidateId, String path, String name, String description,
|
||||
CandidateStatus status, int resourceCount, long totalBytes,
|
||||
List<Issue> issues) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 创建不可变候选。
|
||||
*/
|
||||
public Candidate {
|
||||
issues = issues == null ? List.of() : List.copyOf(issues);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断候选是否可以进入标准包预检。
|
||||
*
|
||||
* @return 可导入时为 true
|
||||
*/
|
||||
public boolean importable() {
|
||||
return status == CandidateStatus.IMPORTABLE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 候选状态。
|
||||
*/
|
||||
public enum CandidateStatus {
|
||||
/** 可进入标准包预检。 */
|
||||
IMPORTABLE,
|
||||
/** 需修正仓库内容后重新扫描。 */
|
||||
NEEDS_ATTENTION
|
||||
}
|
||||
|
||||
/**
|
||||
* 候选校验问题。
|
||||
*
|
||||
* @param code 稳定问题码
|
||||
* @param message 用户可读说明
|
||||
* @param path 问题文件路径
|
||||
*/
|
||||
public record Issue(String code, String message, String path) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import com.alicp.jetcache.AutoReleaseLock;
|
||||
import com.alicp.jetcache.Cache;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.Serial;
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* 保存短期 Git 扫描会话并约束租户、账号与并发使用。
|
||||
*/
|
||||
@Service
|
||||
public class SkillGitScanStore {
|
||||
|
||||
private static final String CACHE_PREFIX = "skill:git-scan:";
|
||||
|
||||
private final Cache<String, Object> defaultCache;
|
||||
private final SkillGitImportProperties properties;
|
||||
|
||||
/**
|
||||
* 创建 Git 扫描会话仓库。
|
||||
*
|
||||
* @param defaultCache 平台默认缓存
|
||||
* @param properties Git 导入配置
|
||||
*/
|
||||
public SkillGitScanStore(@Qualifier("defaultCache") Cache<String, Object> defaultCache,
|
||||
SkillGitImportProperties properties) {
|
||||
this.defaultCache = defaultCache;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建绑定当前账号的扫描会话。
|
||||
*
|
||||
* @param repository 仓库固定提交信息
|
||||
* @param branchRef 默认分支完整引用
|
||||
* @param candidates 候选列表
|
||||
* @return 可返回给前端的扫描结果
|
||||
*/
|
||||
public SkillGitScanResult create(SkillGitScanResult.RepositoryInfo repository,
|
||||
String branchRef,
|
||||
List<SkillGitScanResult.Candidate> candidates) {
|
||||
LoginAccount account = requireAccount();
|
||||
String token = UUID.randomUUID().toString().replace("-", "");
|
||||
Date expiresAt = new Date(System.currentTimeMillis() + properties.getScanTtl().toMillis());
|
||||
int importable = (int) candidates.stream().filter(SkillGitScanResult.Candidate::importable).count();
|
||||
SkillGitScanResult result = new SkillGitScanResult(token, repository,
|
||||
new SkillGitScanResult.Summary(candidates.size(), importable, candidates.size() - importable),
|
||||
candidates, expiresAt);
|
||||
Session session = new Session(account.getTenantId(), account.getId(), branchRef, result);
|
||||
long ttlSeconds = Math.max(1L, properties.getScanTtl().toSeconds());
|
||||
defaultCache.put(cacheKey(token), session, ttlSeconds, TimeUnit.SECONDS);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在扫描会话单次锁内执行业务动作。
|
||||
*
|
||||
* @param token 扫描令牌
|
||||
* @param action 受锁保护的动作
|
||||
* @param <T> 返回值类型
|
||||
* @return 动作结果
|
||||
*/
|
||||
public <T> T withLockedSession(String token, Function<Session, T> action) {
|
||||
validateToken(token);
|
||||
LoginAccount account = requireAccount();
|
||||
try (AutoReleaseLock lock = defaultCache.tryLock(lockKey(token), 5, TimeUnit.MINUTES)) {
|
||||
if (lock == null) {
|
||||
throw new BusinessException("Git 仓库导入正在处理中，请勿重复提交");
|
||||
}
|
||||
return action.apply(findOwned(token, account));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除已成功转换为标准导入预览的扫描会话。
|
||||
*
|
||||
* @param token 扫描令牌
|
||||
*/
|
||||
public void remove(String token) {
|
||||
validateToken(token);
|
||||
defaultCache.remove(cacheKey(token));
|
||||
}
|
||||
|
||||
private Session findOwned(String token, LoginAccount account) {
|
||||
Object cached = defaultCache.get(cacheKey(token));
|
||||
if (!(cached instanceof Session session)) {
|
||||
throw new BusinessException(404, 404, "Git 仓库扫描结果不存在或已过期，请重新扫描");
|
||||
}
|
||||
if (!account.getTenantId().equals(session.tenantId())
|
||||
|| !account.getId().equals(session.accountId())) {
|
||||
throw new BusinessException(403, 403, "无权限使用该 Git 仓库扫描结果");
|
||||
}
|
||||
Date expiresAt = session.result().expiresAt();
|
||||
if (expiresAt == null || !expiresAt.after(new Date())) {
|
||||
defaultCache.remove(cacheKey(token));
|
||||
throw new BusinessException("Git 仓库扫描结果已过期，请重新扫描");
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
private LoginAccount requireAccount() {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
if (account == null || account.getId() == null || account.getTenantId() == null) {
|
||||
throw new BusinessException(401, 401, "未登录或登录态无效");
|
||||
}
|
||||
return account;
|
||||
}
|
||||
|
||||
private void validateToken(String token) {
|
||||
if (token == null || !token.matches("^[a-fA-F0-9]{32}$")) {
|
||||
throw new BusinessException("Git 仓库扫描令牌格式不正确");
|
||||
}
|
||||
}
|
||||
|
||||
private String cacheKey(String token) {
|
||||
return CACHE_PREFIX + token;
|
||||
}
|
||||
|
||||
private String lockKey(String token) {
|
||||
return CACHE_PREFIX + "lock:" + token;
|
||||
}
|
||||
|
||||
/**
|
||||
* 可序列化的短期扫描会话。
|
||||
*
|
||||
* @param tenantId 租户 ID
|
||||
* @param accountId 账号 ID
|
||||
* @param branchRef 默认分支完整引用
|
||||
* @param result 对外扫描结果
|
||||
*/
|
||||
public record Session(java.math.BigInteger tenantId,
|
||||
java.math.BigInteger accountId,
|
||||
String branchRef,
|
||||
SkillGitScanResult result) implements Serializable {
|
||||
|
||||
@Serial
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import tech.easyflow.skill.validation.SkillValidationIssue;
|
||||
public class SkillImportPreview {
|
||||
|
||||
private List<SkillImportPreviewItem> skills = new ArrayList<>();
|
||||
private String sourceName;
|
||||
private String importToken;
|
||||
private Date expiresAt;
|
||||
private List<SkillValidationIssue> issues = new ArrayList<>();
|
||||
@@ -33,6 +34,24 @@ public class SkillImportPreview {
|
||||
this.skills = skills == null ? new ArrayList<>() : skills;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取本预览对应的原始导入包名称。
|
||||
*
|
||||
* @return 原始导入包名称
|
||||
*/
|
||||
public String getSourceName() {
|
||||
return sourceName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置本预览对应的原始导入包名称。
|
||||
*
|
||||
* @param sourceName 原始导入包名称
|
||||
*/
|
||||
public void setSourceName(String sourceName) {
|
||||
this.sourceName = sourceName;
|
||||
}
|
||||
|
||||
public String getImportToken() { return importToken; }
|
||||
public void setImportToken(String importToken) { this.importToken = importToken; }
|
||||
public Date getExpiresAt() { return expiresAt; }
|
||||
|
||||
@@ -2,6 +2,7 @@ package tech.easyflow.skill.imports;
|
||||
|
||||
import tech.easyflow.skill.entity.Skill;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@@ -10,6 +11,17 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
*/
|
||||
public interface SkillImportService {
|
||||
|
||||
/** 单次批量预检或确认允许处理的最大 Skill 数量。 */
|
||||
int MAX_BATCH_SKILL_COUNT = 50;
|
||||
|
||||
/**
|
||||
* 批量预检上传的标准 Skill ZIP，并将多 Skill 包拆分为独立预览。
|
||||
*
|
||||
* @param files 标准 Skill ZIP 列表
|
||||
* @return 每个 Skill 对应一个 importToken 的预览列表
|
||||
*/
|
||||
List<SkillImportPreview> previewBatch(List<MultipartFile> files);
|
||||
|
||||
/**
|
||||
* 上传并创建可单次确认的导入预览。
|
||||
*
|
||||
@@ -18,6 +30,15 @@ public interface SkillImportService {
|
||||
*/
|
||||
SkillImportPreview preview(MultipartFile file);
|
||||
|
||||
/**
|
||||
* 暂存并预检后端生成的标准 Skill ZIP。
|
||||
*
|
||||
* @param originalName 用于结果展示的原始文件名
|
||||
* @param file 后端受控临时 ZIP
|
||||
* @return 导入预览与 importToken
|
||||
*/
|
||||
SkillImportPreview preview(String originalName, File file);
|
||||
|
||||
/**
|
||||
* 使用单次 importToken 确认导入。
|
||||
*
|
||||
|
||||
@@ -2,9 +2,12 @@ package tech.easyflow.skill.imports;
|
||||
|
||||
import com.easyagents.skill.codec.SkillPackageReadOptions;
|
||||
import com.easyagents.skill.codec.SkillPackageReadResult;
|
||||
import com.easyagents.skill.codec.SkillPackageWriteOptions;
|
||||
import com.easyagents.skill.codec.ZipSkillPackageCodec;
|
||||
import com.easyagents.skill.exception.SkillPackageException;
|
||||
import com.easyagents.skill.model.SkillDocument;
|
||||
import com.easyagents.skill.model.SkillPackage;
|
||||
import com.easyagents.skill.model.SkillPackageLayout;
|
||||
import com.easyagents.skill.model.SkillPackageLimits;
|
||||
import com.easyagents.skill.model.SkillResourceKind;
|
||||
import com.easyagents.skill.store.SkillContentStage;
|
||||
@@ -38,10 +41,15 @@ import tech.easyflow.system.enums.ResourceAction;
|
||||
import tech.easyflow.system.enums.VisibilityScope;
|
||||
import tech.easyflow.system.service.ResourceAccessService;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
@@ -56,7 +64,7 @@ import java.util.Set;
|
||||
public class SkillImportServiceImpl implements SkillImportService {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SkillImportServiceImpl.class);
|
||||
private static final int MAX_RENAMES = 20;
|
||||
private static final int MAX_RENAMES = MAX_BATCH_SKILL_COUNT;
|
||||
private static final int NAME_CONFLICT_ERROR_CODE = 4092;
|
||||
private static final String NAME_UNAVAILABLE_REASON = "NAME_UNAVAILABLE";
|
||||
|
||||
@@ -99,6 +107,117 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
return hasExactlyOneSkill(decoded) ? buildPreview(decoded, null) : invalidSkillCountPreview();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public List<SkillImportPreview> previewBatch(List<MultipartFile> files) {
|
||||
if (files == null || files.isEmpty()) {
|
||||
throw new BusinessException("请选择至少一个 Skill ZIP 文件");
|
||||
}
|
||||
if (files.size() > MAX_BATCH_SKILL_COUNT) {
|
||||
throw batchLimitExceeded();
|
||||
}
|
||||
|
||||
List<SkillImportPreview> prepared = new ArrayList<>();
|
||||
int totalSkills = 0;
|
||||
try {
|
||||
for (MultipartFile file : files) {
|
||||
PreparedUpload upload = previewUploadedPackage(
|
||||
file, MAX_BATCH_SKILL_COUNT - totalSkills);
|
||||
prepared.addAll(upload.previews());
|
||||
totalSkills += upload.skillCount();
|
||||
}
|
||||
return List.copyOf(prepared);
|
||||
} catch (RuntimeException exception) {
|
||||
cancelPreparedPreviews(prepared);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 预检一个上传包，并在需要时拆分为独立 Skill 预览。
|
||||
*
|
||||
* @param file 上传的标准 ZIP
|
||||
* @param remainingSkills 当前批次剩余可接收 Skill 数
|
||||
* @return 本包产生的独立预览及 Skill 数量
|
||||
*/
|
||||
private PreparedUpload previewUploadedPackage(MultipartFile file, int remainingSkills) {
|
||||
validateUpload(file);
|
||||
String sourceName = file.getOriginalFilename();
|
||||
SkillPackageReadResult reportOnly;
|
||||
try (InputStream input = file.getInputStream()) {
|
||||
reportOnly = new ZipSkillPackageCodec(new PreviewContentStore())
|
||||
.decode(input, SkillPackageReadOptions.reportOnly());
|
||||
} catch (SkillPackageException exception) {
|
||||
return new PreparedUpload(List.of(failedPreview(exception, sourceName)), 0);
|
||||
} catch (IOException exception) {
|
||||
LOG.error("读取标准 Skill ZIP 失败，file={}", sourceName, exception);
|
||||
throw new BusinessException(500, 500, "读取 Skill 导入包失败", exception);
|
||||
}
|
||||
|
||||
int skillCount = reportOnly.getSkillPackage().getSkills().size();
|
||||
if (skillCount > remainingSkills) {
|
||||
throw batchLimitExceeded();
|
||||
}
|
||||
if (skillCount == 1) {
|
||||
SkillImportPreview preview = preview(file);
|
||||
preview.setSourceName(sourceName);
|
||||
return new PreparedUpload(List.of(preview), 1);
|
||||
}
|
||||
if (reportOnly.getValidationReport().hasErrors()) {
|
||||
return new PreparedUpload(List.of(failedPreview(reportOnly, sourceName)), skillCount);
|
||||
}
|
||||
return splitAndPreview(file, sourceName, skillCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将已通过整包校验的多 Skill ZIP 拆成独立标准包并逐一暂存。
|
||||
*
|
||||
* @param file 原始上传 ZIP
|
||||
* @param sourceName 原始文件名
|
||||
* @param skillCount 已解析的 Skill 数量
|
||||
* @return 每个 Skill 对应一个 token 的预览结果
|
||||
*/
|
||||
private PreparedUpload splitAndPreview(MultipartFile file, String sourceName, int skillCount) {
|
||||
List<SkillImportPreview> prepared = new ArrayList<>(skillCount);
|
||||
try (ZipSkillPackageCodec codec = new ZipSkillPackageCodec();
|
||||
InputStream input = file.getInputStream()) {
|
||||
SkillPackageReadResult decoded = codec.decode(input, SkillPackageReadOptions.defaults());
|
||||
for (com.easyagents.skill.model.Skill skill : decoded.getSkillPackage().getSkills()) {
|
||||
Path archive = null;
|
||||
try {
|
||||
archive = Files.createTempFile("easyflow-skill-import-", ".zip");
|
||||
try (OutputStream output = Files.newOutputStream(archive,
|
||||
StandardOpenOption.TRUNCATE_EXISTING)) {
|
||||
codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)),
|
||||
output, SkillPackageWriteOptions.defaults());
|
||||
}
|
||||
SkillImportPreview preview = preview(sourceName, archive.toFile());
|
||||
if (!hasUsableSingleSkillPreview(preview)) {
|
||||
cancelPreview(preview);
|
||||
throw new BusinessException(500, 500, "拆分 Skill 导入包后预检失败，请重新打包后重试");
|
||||
}
|
||||
preview.setSourceName(sourceName);
|
||||
prepared.add(preview);
|
||||
} finally {
|
||||
deleteLocalArchive(archive);
|
||||
}
|
||||
}
|
||||
return new PreparedUpload(List.copyOf(prepared), skillCount);
|
||||
} catch (SkillPackageException exception) {
|
||||
cancelPreparedPreviews(prepared);
|
||||
return new PreparedUpload(List.of(failedPreview(exception, sourceName)), skillCount);
|
||||
} catch (IOException exception) {
|
||||
cancelPreparedPreviews(prepared);
|
||||
LOG.error("拆分多 Skill ZIP 失败，file={}", sourceName, exception);
|
||||
throw new BusinessException(500, 500, "拆分 Skill 导入包失败", exception);
|
||||
} catch (RuntimeException exception) {
|
||||
cancelPreparedPreviews(prepared);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@@ -109,24 +228,10 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
String storedPath = null;
|
||||
try {
|
||||
storedPath = fileStorageService.save(file, "skill-imports/" + account.getTenantId());
|
||||
if (storedPath == null || storedPath.isBlank()) {
|
||||
throw new BusinessException(500, 500, "Skill 导入临时包存储失败，请稍后重试");
|
||||
}
|
||||
SkillPackageReadResult decoded;
|
||||
try (InputStream input = fileStorageService.readStream(storedPath)) {
|
||||
decoded = new ZipSkillPackageCodec(new PreviewContentStore())
|
||||
.decode(input, SkillPackageReadOptions.reportOnly());
|
||||
}
|
||||
if (!hasExactlyOneSkill(decoded)) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
return invalidSkillCountPreview();
|
||||
}
|
||||
SkillImportStage stage = stageStore.create(storedPath, file.getOriginalFilename());
|
||||
SkillImportPreview preview = buildPreview(decoded, stage);
|
||||
return preview;
|
||||
return previewStoredPath(storedPath, file.getOriginalFilename());
|
||||
} catch (SkillPackageException exception) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
return failedPreview(exception);
|
||||
return failedPreview(exception, file.getOriginalFilename());
|
||||
} catch (BusinessException exception) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
throw exception;
|
||||
@@ -140,6 +245,50 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public SkillImportPreview preview(String originalName, File file) {
|
||||
validateGeneratedFile(originalName, file);
|
||||
LoginAccount account = requireAccount();
|
||||
String storedPath = null;
|
||||
try {
|
||||
storedPath = fileStorageService.save(file, "skill-imports/" + account.getTenantId());
|
||||
return previewStoredPath(storedPath, originalName);
|
||||
} catch (SkillPackageException exception) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
return failedPreview(exception, originalName);
|
||||
} catch (BusinessException exception) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
throw exception;
|
||||
} catch (IOException exception) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
LOG.error("读取后端生成的标准 Skill ZIP 失败，path={}", storedPath, exception);
|
||||
throw new BusinessException(500, 500, "读取 Skill 导入包失败", exception);
|
||||
} catch (RuntimeException exception) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private SkillImportPreview previewStoredPath(String storedPath, String originalName) throws IOException {
|
||||
if (storedPath == null || storedPath.isBlank()) {
|
||||
throw new BusinessException(500, 500, "Skill 导入临时包存储失败，请稍后重试");
|
||||
}
|
||||
SkillPackageReadResult decoded;
|
||||
try (InputStream input = fileStorageService.readStream(storedPath)) {
|
||||
decoded = new ZipSkillPackageCodec(new PreviewContentStore())
|
||||
.decode(input, SkillPackageReadOptions.reportOnly());
|
||||
}
|
||||
if (!hasExactlyOneSkill(decoded)) {
|
||||
cleanupUnregisteredPath(storedPath);
|
||||
return invalidSkillCountPreview(originalName);
|
||||
}
|
||||
SkillImportStage stage = stageStore.create(storedPath, originalName);
|
||||
return buildPreview(decoded, stage);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@@ -236,6 +385,7 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
private SkillImportPreview buildPreview(SkillPackageReadResult decoded, SkillImportStage stage) {
|
||||
SkillImportPreview preview = new SkillImportPreview();
|
||||
if (stage != null) {
|
||||
preview.setSourceName(stage.getOriginalName());
|
||||
preview.setImportToken(stage.getImportToken());
|
||||
preview.setExpiresAt(stage.getExpiresAt());
|
||||
}
|
||||
@@ -376,6 +526,19 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
}
|
||||
}
|
||||
|
||||
private void validateGeneratedFile(String originalName, File file) {
|
||||
if (file == null || !file.isFile() || file.length() == 0L) {
|
||||
throw new BusinessException("Skill 导入文件不能为空");
|
||||
}
|
||||
long limit = SkillPackageLimits.defaults().getMaxCompressedPackageBytes();
|
||||
if (file.length() > limit) {
|
||||
throw new BusinessException(413, 4131, "Skill 导入文件超过 " + limit + " 字节限制");
|
||||
}
|
||||
if (originalName == null || !originalName.toLowerCase(java.util.Locale.ROOT).endsWith(".zip")) {
|
||||
throw new BusinessException("Skill 导入仅支持标准 .zip 文件");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateConfirmRequest(SkillImportConfirmRequest request) {
|
||||
if (request == null || request.getImportToken() == null || request.getImportToken().isBlank()) {
|
||||
throw new BusinessException("Skill 导入确认参数不能为空");
|
||||
@@ -399,14 +562,33 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
}
|
||||
|
||||
private SkillImportPreview invalidSkillCountPreview() {
|
||||
return invalidSkillCountPreview(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建单 token 路径收到非单 Skill 包时的失败预览。
|
||||
*
|
||||
* @param sourceName 原始文件名
|
||||
* @return 失败预览
|
||||
*/
|
||||
private SkillImportPreview invalidSkillCountPreview(String sourceName) {
|
||||
SkillImportPreview preview = new SkillImportPreview();
|
||||
preview.setSourceName(sourceName);
|
||||
preview.setIssues(List.of(SkillValidationIssue.of(
|
||||
"ERROR", "STANDARD_PACKAGE_SKILL_COUNT", "每个标准 Skill ZIP 必须且只能包含一个 Skill", null)));
|
||||
return preview;
|
||||
}
|
||||
|
||||
private SkillImportPreview failedPreview(SkillPackageException exception) {
|
||||
/**
|
||||
* 将包解析异常转换为可展示的失败预览。
|
||||
*
|
||||
* @param exception 包解析异常
|
||||
* @param sourceName 原始文件名
|
||||
* @return 失败预览
|
||||
*/
|
||||
private SkillImportPreview failedPreview(SkillPackageException exception, String sourceName) {
|
||||
SkillImportPreview preview = new SkillImportPreview();
|
||||
preview.setSourceName(sourceName);
|
||||
preview.setSkills(List.of());
|
||||
if (exception.getReport() == null) {
|
||||
preview.setIssues(List.of(SkillValidationIssue.of(
|
||||
@@ -424,6 +606,92 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
return preview;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将报告模式的整包校验错误转换为失败预览。
|
||||
*
|
||||
* @param decoded 报告模式解码结果
|
||||
* @param sourceName 原始文件名
|
||||
* @return 失败预览
|
||||
*/
|
||||
private SkillImportPreview failedPreview(SkillPackageReadResult decoded, String sourceName) {
|
||||
SkillImportPreview preview = new SkillImportPreview();
|
||||
preview.setSourceName(sourceName);
|
||||
preview.setSkills(List.of());
|
||||
preview.setIssues(decoded.getValidationReport().getIssues().stream().map(source -> {
|
||||
SkillValidationIssue issue = SkillValidationIssue.of(source.getSeverity().name(), source.getCode(),
|
||||
source.getMessage(), source.getPath());
|
||||
issue.setLine(source.getLine());
|
||||
issue.setColumn(source.getColumn());
|
||||
issue.setSuggestion(source.getSuggestion());
|
||||
return issue;
|
||||
}).toList());
|
||||
return preview;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断拆分后的预览是否满足单 Skill token 不变量。
|
||||
*
|
||||
* @param preview 拆分包预览
|
||||
* @return token、Skill 与校验结果均可用时返回 true
|
||||
*/
|
||||
private boolean hasUsableSingleSkillPreview(SkillImportPreview preview) {
|
||||
return preview != null
|
||||
&& preview.getImportToken() != null
|
||||
&& !preview.getImportToken().isBlank()
|
||||
&& preview.getSkills().size() == 1
|
||||
&& preview.getIssues().stream().noneMatch(issue -> "ERROR".equals(issue.getSeverity()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建统一的批量 Skill 数量超限异常。
|
||||
*
|
||||
* @return 业务异常
|
||||
*/
|
||||
private BusinessException batchLimitExceeded() {
|
||||
return new BusinessException("单次最多导入 " + MAX_BATCH_SKILL_COUNT + " 个 Skill");
|
||||
}
|
||||
|
||||
/**
|
||||
* 尽力回滚一组已创建的预览 token。
|
||||
*
|
||||
* @param previews 已创建的预览
|
||||
*/
|
||||
private void cancelPreparedPreviews(List<SkillImportPreview> previews) {
|
||||
previews.forEach(this::cancelPreview);
|
||||
}
|
||||
|
||||
/**
|
||||
* 尽力回滚一个已创建的预览 token。
|
||||
*
|
||||
* @param preview 已创建的预览
|
||||
*/
|
||||
private void cancelPreview(SkillImportPreview preview) {
|
||||
if (preview == null || preview.getImportToken() == null || preview.getImportToken().isBlank()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
cancel(preview.getImportToken());
|
||||
} catch (RuntimeException cleanupException) {
|
||||
LOG.error("回滚 Skill 导入预览失败，token={}", preview.getImportToken(), cleanupException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 尽力删除拆包过程中创建的本地临时 ZIP。
|
||||
*
|
||||
* @param archive 本地临时 ZIP 路径
|
||||
*/
|
||||
private void deleteLocalArchive(Path archive) {
|
||||
if (archive == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(archive);
|
||||
} catch (IOException cleanupException) {
|
||||
LOG.warn("清理本地 Skill 拆分临时包失败，path={}", archive, cleanupException);
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupUnregisteredPath(String path) {
|
||||
if (path == null || path.isBlank()) {
|
||||
return;
|
||||
@@ -500,4 +768,13 @@ public class SkillImportServiceImpl implements SkillImportService {
|
||||
@Override public InputStream open(String contentRef) { throw new UnsupportedOperationException(); }
|
||||
@Override public boolean exists(String contentRef) { return true; }
|
||||
}
|
||||
|
||||
/**
|
||||
* 单个上传包完成预检后的独立预览及实际 Skill 数量。
|
||||
*
|
||||
* @param previews 独立预览列表
|
||||
* @param skillCount 实际解析出的 Skill 数量
|
||||
*/
|
||||
private record PreparedUpload(List<SkillImportPreview> previews, int skillCount) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
/**
|
||||
* {@link GitRepositoryAccessPolicy} 地址安全契约测试。
|
||||
*/
|
||||
public class GitRepositoryAccessPolicyTest {
|
||||
|
||||
private GitRepositoryAccessPolicy policy;
|
||||
|
||||
/**
|
||||
* 初始化不依赖公网 DNS 的可信测试主机。
|
||||
*/
|
||||
@Before
|
||||
public void setUp() {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
properties.setTrustedPrivateHosts(Set.of("git.example.test"));
|
||||
policy = new GitRepositoryAccessPolicy(properties);
|
||||
}
|
||||
|
||||
/**
|
||||
* 仓库地址允许省略 .git 后缀并移除结尾斜线。
|
||||
*/
|
||||
@Test
|
||||
public void acceptsHttpsRepositoryWithOptionalGitSuffix() {
|
||||
assertEquals(policy.normalizeRepositoryUrl(
|
||||
" https://git.example.test/team/skills/ "),
|
||||
"https://git.example.test/team/skills");
|
||||
assertEquals(policy.normalizeRepositoryUrl(
|
||||
"https://git.example.test/team/skills.git"),
|
||||
"https://git.example.test/team/skills.git");
|
||||
assertEquals(policy.normalizeRepositoryUrl("https://192.0.1.1/team/skills.git"),
|
||||
"https://192.0.1.1/team/skills.git");
|
||||
}
|
||||
|
||||
/**
|
||||
* 拒绝非 HTTPS、内嵌凭证和 URL 参数。
|
||||
*/
|
||||
@Test
|
||||
public void rejectsUnsafeRepositoryInputs() {
|
||||
assertRejected("http://git.example.test/team/skills.git");
|
||||
assertRejected("https://user:secret@git.example.test/team/skills.git");
|
||||
assertRejected("https://git.example.test/team/skills.git?token=secret");
|
||||
assertRejected("https://git.example.test:8443/team/skills.git");
|
||||
assertRejected("https://127.0.0.1/team/skills.git");
|
||||
}
|
||||
|
||||
/**
|
||||
* 实际连接使用策略校验后返回的固定 DNS 地址。
|
||||
*
|
||||
* @throws Exception 构造测试地址失败
|
||||
*/
|
||||
@Test
|
||||
public void resolvesAddressesForPinnedConnection() throws Exception {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
InetAddress publicAddress = InetAddress.getByAddress(new byte[]{93, (byte) 184, (byte) 216, 34});
|
||||
GitRepositoryAccessPolicy pinnedPolicy = new GitRepositoryAccessPolicy(
|
||||
properties, host -> new InetAddress[]{publicAddress});
|
||||
|
||||
GitRepositoryAccessPolicy.ResolvedConnection resolved = pinnedPolicy.resolveConnection(
|
||||
new URL("https://git.example.test/team/skills.git/info/refs"));
|
||||
|
||||
assertEquals("git.example.test", resolved.host());
|
||||
assertEquals(publicAddress, resolved.addresses()[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 输入校验后主机重新绑定到内网时，实际连接阶段必须拒绝。
|
||||
*
|
||||
* @throws Exception 构造测试地址失败
|
||||
*/
|
||||
@Test
|
||||
public void rejectsDnsRebindingBeforeConnection() throws Exception {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
InetAddress publicAddress = InetAddress.getByAddress(new byte[]{93, (byte) 184, (byte) 216, 34});
|
||||
InetAddress loopback = InetAddress.getByAddress(new byte[]{127, 0, 0, 1});
|
||||
AtomicInteger resolutions = new AtomicInteger();
|
||||
GitRepositoryAccessPolicy rebindingPolicy = new GitRepositoryAccessPolicy(properties,
|
||||
host -> resolutions.getAndIncrement() == 0
|
||||
? new InetAddress[]{publicAddress} : new InetAddress[]{loopback});
|
||||
|
||||
assertEquals("https://git.example.test/team/skills.git",
|
||||
rebindingPolicy.normalizeRepositoryUrl("https://git.example.test/team/skills.git"));
|
||||
assertThrows(IOException.class, () -> rebindingPolicy.resolveConnection(
|
||||
new URL("https://git.example.test/team/skills.git/info/refs")));
|
||||
}
|
||||
|
||||
private void assertRejected(String value) {
|
||||
assertThrows(BusinessException.class, () -> policy.normalizeRepositoryUrl(value));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.eclipse.jgit.api.Git;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
/**
|
||||
* {@link SkillGitCandidateScanner} 固定提交扫描与标准包组装测试。
|
||||
*/
|
||||
public class SkillGitCandidateScannerTest {
|
||||
|
||||
/**
|
||||
* 发现根目录和嵌套 Skill，并把不可解析的 LFS 资源标为需处理。
|
||||
*
|
||||
* @throws Exception 仓库或 ZIP 读写失败
|
||||
*/
|
||||
@Test
|
||||
public void discoversCandidatesAndCreatesIndependentStandardZip() throws Exception {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
SkillGitCandidateScanner scanner = new SkillGitCandidateScanner(properties);
|
||||
Path directory = Files.createTempDirectory("skill-git-scanner-test-");
|
||||
try (Git git = Git.init().setDirectory(directory.toFile()).call()) {
|
||||
write(directory.resolve("SKILL.md"), skill("root-skill", "根目录技能"));
|
||||
write(directory.resolve("references/guide.md"), "# Guide\n");
|
||||
write(directory.resolve("nested-skill/SKILL.md"), skill("nested-skill", "嵌套技能"));
|
||||
write(directory.resolve("nested-skill/assets/model.bin"),
|
||||
"version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 10\n");
|
||||
git.add().addFilepattern(".").call();
|
||||
String commitSha = git.commit().setMessage("test")
|
||||
.setAuthor("EasyFlow", "test@easyflow.local")
|
||||
.setCommitter("EasyFlow", "test@easyflow.local").call().getName();
|
||||
SkillGitRepositoryReader.Snapshot snapshot = new SkillGitRepositoryReader.Snapshot(
|
||||
git.getRepository(), directory, "refs/heads/master", commitSha);
|
||||
|
||||
SkillGitRepositoryScan scan = scanner.scan(snapshot);
|
||||
|
||||
assertEquals(scan.candidates().size(), 2);
|
||||
SkillGitScanResult.Candidate root = candidate(scan, "root-skill");
|
||||
SkillGitScanResult.Candidate nested = candidate(scan, "nested-skill");
|
||||
assertTrue(root.importable());
|
||||
assertFalse(nested.importable());
|
||||
assertTrue(nested.issues().stream()
|
||||
.anyMatch(issue -> "GIT_LFS_UNRESOLVED".equals(issue.code())));
|
||||
|
||||
Path archive = scanner.createStandardZip(snapshot, scan, root.candidateId());
|
||||
try {
|
||||
assertEquals(zipEntries(archive),
|
||||
Set.of("root-skill/SKILL.md", "root-skill/references/guide.md"));
|
||||
} finally {
|
||||
scanner.deleteArchive(archive);
|
||||
}
|
||||
// Snapshot 与 Git 共享 Repository，交给 Git 的 try-with-resources 统一关闭。
|
||||
} finally {
|
||||
deleteDirectory(directory);
|
||||
}
|
||||
}
|
||||
|
||||
private SkillGitScanResult.Candidate candidate(SkillGitRepositoryScan scan, String name) {
|
||||
return scan.candidates().stream().filter(item -> name.equals(item.name())).findFirst()
|
||||
.orElseThrow();
|
||||
}
|
||||
|
||||
private Set<String> zipEntries(Path archive) throws IOException {
|
||||
Set<String> entries = new HashSet<>();
|
||||
try (ZipInputStream input = new ZipInputStream(Files.newInputStream(archive))) {
|
||||
java.util.zip.ZipEntry entry;
|
||||
while ((entry = input.getNextEntry()) != null) {
|
||||
entries.add(entry.getName());
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private String skill(String name, String description) {
|
||||
return "---\nname: " + name + "\ndescription: " + description + "\n---\n# " + name + "\n";
|
||||
}
|
||||
|
||||
private void write(Path path, String content) throws IOException {
|
||||
Files.createDirectories(path.getParent());
|
||||
Files.writeString(path, content, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private void deleteDirectory(Path directory) throws IOException {
|
||||
if (!Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
try (java.util.stream.Stream<Path> paths = Files.walk(directory)) {
|
||||
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
|
||||
Files.deleteIfExists(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
/**
|
||||
* {@link SkillGitOperationExecutor} 并发与总超时契约测试。
|
||||
*/
|
||||
public class SkillGitOperationExecutorTest {
|
||||
|
||||
private SkillGitOperationExecutor operationExecutor;
|
||||
|
||||
/**
|
||||
* 关闭测试中创建的远程操作执行器。
|
||||
*/
|
||||
@After
|
||||
public void tearDown() {
|
||||
if (operationExecutor != null) {
|
||||
operationExecutor.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 单实例并发达到上限时快速拒绝额外任务。
|
||||
*
|
||||
* @throws Exception 线程协作失败
|
||||
*/
|
||||
@Test
|
||||
public void rejectsOperationWhenConcurrencyIsExhausted() throws Exception {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
properties.setMaxConcurrentOperations(1);
|
||||
operationExecutor = new SkillGitOperationExecutor(properties);
|
||||
CountDownLatch started = new CountDownLatch(1);
|
||||
CountDownLatch release = new CountDownLatch(1);
|
||||
ExecutorService caller = Executors.newSingleThreadExecutor();
|
||||
try {
|
||||
Future<String> first = caller.submit(() -> operationExecutor.execute(() -> {
|
||||
started.countDown();
|
||||
release.await();
|
||||
return "done";
|
||||
}));
|
||||
assertTrue(started.await(2, TimeUnit.SECONDS));
|
||||
|
||||
BusinessException exception = assertThrows(BusinessException.class,
|
||||
() -> operationExecutor.execute(() -> "second"));
|
||||
|
||||
assertEquals(429, exception.getHttpStatus());
|
||||
release.countDown();
|
||||
assertEquals("done", first.get(2, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
release.countDown();
|
||||
caller.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 超过远程操作总时限时返回网关超时并取消任务。
|
||||
*/
|
||||
@Test
|
||||
public void cancelsOperationAfterTotalTimeout() {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
properties.setOperationTimeout(Duration.ofMillis(30));
|
||||
operationExecutor = new SkillGitOperationExecutor(properties);
|
||||
|
||||
BusinessException exception = assertThrows(BusinessException.class,
|
||||
() -> operationExecutor.execute(() -> {
|
||||
Thread.sleep(5_000L);
|
||||
return "late";
|
||||
}));
|
||||
|
||||
assertEquals(504, exception.getHttpStatus());
|
||||
}
|
||||
|
||||
/**
|
||||
* 任务忽略中断并在超时后返回资源时，执行器负责回收该结果。
|
||||
*
|
||||
* @throws Exception 等待清理回调失败
|
||||
*/
|
||||
@Test
|
||||
public void cleansResultProducedAfterTimeout() throws Exception {
|
||||
SkillGitImportProperties properties = new SkillGitImportProperties();
|
||||
properties.setOperationTimeout(Duration.ofMillis(30));
|
||||
operationExecutor = new SkillGitOperationExecutor(properties);
|
||||
CountDownLatch cleaned = new CountDownLatch(1);
|
||||
|
||||
assertThrows(BusinessException.class, () -> operationExecutor.execute(() -> {
|
||||
try {
|
||||
Thread.sleep(5_000L);
|
||||
} catch (InterruptedException ignored) {
|
||||
// 模拟底层驱动在取消后仍返回已经创建的资源。
|
||||
}
|
||||
return "late-resource";
|
||||
}, result -> cleaned.countDown()));
|
||||
|
||||
assertTrue(cleaned.await(2, TimeUnit.SECONDS));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package tech.easyflow.skill.gitimport;
|
||||
|
||||
import com.alicp.jetcache.Cache;
|
||||
import com.alicp.jetcache.CacheResult;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link SkillGitScanRateLimiter} 分布式频控契约测试。
|
||||
*/
|
||||
public class SkillGitScanRateLimiterTest {
|
||||
|
||||
/**
|
||||
* 首次扫描写入带过期时间的账号频控键。
|
||||
*/
|
||||
@Test
|
||||
public void acceptsFirstScanAndStoresAccountScopedKey() {
|
||||
Cache<String, Object> cache = mock(Cache.class);
|
||||
when(cache.PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE),
|
||||
anyLong(), eq(TimeUnit.MILLISECONDS))).thenReturn(CacheResult.SUCCESS_WITHOUT_MSG);
|
||||
SkillGitScanRateLimiter limiter = new SkillGitScanRateLimiter(cache, new SkillGitImportProperties());
|
||||
|
||||
limiter.check(account());
|
||||
|
||||
verify(cache).PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE),
|
||||
eq(2_000L), eq(TimeUnit.MILLISECONDS));
|
||||
}
|
||||
|
||||
/**
|
||||
* 频控键已存在时返回 HTTP 429。
|
||||
*/
|
||||
@Test
|
||||
public void rejectsRepeatedScanWithinMinimumInterval() {
|
||||
Cache<String, Object> cache = mock(Cache.class);
|
||||
when(cache.PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE),
|
||||
anyLong(), eq(TimeUnit.MILLISECONDS))).thenReturn(CacheResult.EXISTS_WITHOUT_MSG);
|
||||
SkillGitScanRateLimiter limiter = new SkillGitScanRateLimiter(cache, new SkillGitImportProperties());
|
||||
|
||||
BusinessException exception = assertThrows(BusinessException.class,
|
||||
() -> limiter.check(account()));
|
||||
|
||||
assertEquals(429, exception.getHttpStatus());
|
||||
}
|
||||
|
||||
private LoginAccount account() {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setTenantId(BigInteger.TEN);
|
||||
account.setId(BigInteger.valueOf(20));
|
||||
return account;
|
||||
}
|
||||
}
|
||||
@@ -9,17 +9,24 @@ import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.skill.entity.Skill;
|
||||
import tech.easyflow.skill.entity.SkillImportStage;
|
||||
import tech.easyflow.skill.service.SkillService;
|
||||
import tech.easyflow.skill.store.DBSkillContentStore;
|
||||
import tech.easyflow.system.service.ResourceAccessService;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
@@ -29,8 +36,11 @@ import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
@@ -60,22 +70,66 @@ public class StandardSkillPackageContractTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 一个上传 ZIP 只能承载一个 Skill，批量导入由多个 token 独立完成。
|
||||
* 一个父目录包装的多 Skill ZIP 会被拆成多个独立 token 预览。
|
||||
*/
|
||||
@Test
|
||||
public void previewRejectsMultipleSkillsInOneZip() {
|
||||
SkillService skillService = mock(SkillService.class);
|
||||
when(skillService.list(any(QueryWrapper.class))).thenReturn(List.of());
|
||||
SkillImportServiceImpl service = importService(skillService);
|
||||
public void previewSplitsParentWrappedSkillsIntoIndependentTokens() {
|
||||
BatchFixture fixture = batchImportService();
|
||||
MultipartFile file = multipartFile("skill-bundle.zip", multiSkillZip("skill-bundle/", 2));
|
||||
|
||||
SkillImportPreview preview;
|
||||
List<SkillImportPreview> previews;
|
||||
try (MockedStatic<SaTokenUtil> ignored = login()) {
|
||||
preview = service.previewStandardForTest(new ByteArrayInputStream(multiSkillZip()));
|
||||
previews = fixture.service().previewBatch(List.of(file));
|
||||
}
|
||||
|
||||
assertTrue(preview.getSkills().isEmpty());
|
||||
assertTrue(preview.getIssues().stream()
|
||||
.anyMatch(issue -> "STANDARD_PACKAGE_SKILL_COUNT".equals(issue.getCode())));
|
||||
assertEquals(2, previews.size());
|
||||
assertEquals(List.of("skill-1", "skill-2"), previews.stream()
|
||||
.map(preview -> preview.getSkills().get(0).getName()).toList());
|
||||
assertTrue(previews.stream().allMatch(preview -> preview.getSkills().size() == 1));
|
||||
assertTrue(previews.stream().allMatch(preview -> preview.getImportToken() != null));
|
||||
assertTrue(previews.stream().allMatch(preview -> "skill-bundle.zip".equals(preview.getSourceName())));
|
||||
verify(fixture.fileStorageService(), times(2)).save(any(java.io.File.class), anyString());
|
||||
verify(fixture.stageStore(), times(2)).create(anyString(), anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 单个 ZIP 解析出的 Skill 超过统一上限时，在创建暂存 token 前拒绝。
|
||||
*/
|
||||
@Test
|
||||
public void previewRejectsMoreThanFiftySkills() {
|
||||
BatchFixture fixture = batchImportService();
|
||||
MultipartFile file = multipartFile("too-many.zip", multiSkillZip("", 51));
|
||||
|
||||
BusinessException exception;
|
||||
try (MockedStatic<SaTokenUtil> ignored = login()) {
|
||||
exception = assertThrows(BusinessException.class,
|
||||
() -> fixture.service().previewBatch(List.of(file)));
|
||||
}
|
||||
|
||||
assertTrue(exception.getMessage().contains("50"));
|
||||
verify(fixture.fileStorageService(), times(0)).save(any(java.io.File.class), anyString());
|
||||
verify(fixture.stageStore(), times(0)).create(anyString(), anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 多个上传包合计超过五十个 Skill 时，已创建的独立 token 会全部回滚。
|
||||
*/
|
||||
@Test
|
||||
public void previewEnforcesFiftySkillLimitAcrossUploadedPackages() {
|
||||
BatchFixture fixture = batchImportService();
|
||||
MultipartFile first = multipartFile("first.zip", multiSkillZip("", 25));
|
||||
MultipartFile second = multipartFile("second.zip", multiSkillZip("bundle/", 26));
|
||||
|
||||
BusinessException exception;
|
||||
try (MockedStatic<SaTokenUtil> ignored = login()) {
|
||||
exception = assertThrows(BusinessException.class,
|
||||
() -> fixture.service().previewBatch(List.of(first, second)));
|
||||
}
|
||||
|
||||
assertTrue(exception.getMessage().contains("50"));
|
||||
verify(fixture.fileStorageService(), times(25)).save(any(java.io.File.class), anyString());
|
||||
verify(fixture.stageStore(), times(25)).create(anyString(), anyString());
|
||||
verify(fixture.stageStore(), times(25)).cancel(anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -151,12 +205,21 @@ public class StandardSkillPackageContractTest {
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] multiSkillZip() {
|
||||
/**
|
||||
* 创建包含指定数量 Skill 的测试 ZIP。
|
||||
*
|
||||
* @param parent 可选父目录前缀
|
||||
* @param count Skill 数量
|
||||
* @return ZIP 字节
|
||||
*/
|
||||
private byte[] multiSkillZip(String parent, int count) {
|
||||
try {
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zip = new ZipOutputStream(bytes, StandardCharsets.UTF_8)) {
|
||||
write(zip, "alpha-skill/SKILL.md", skillContent("alpha-skill"));
|
||||
write(zip, "beta-skill/SKILL.md", skillContent("beta-skill"));
|
||||
for (int index = 1; index <= count; index++) {
|
||||
String name = "skill-" + index;
|
||||
write(zip, parent + name + "/SKILL.md", skillContent(name));
|
||||
}
|
||||
}
|
||||
return bytes.toByteArray();
|
||||
} catch (Exception exception) {
|
||||
@@ -164,6 +227,64 @@ public class StandardSkillPackageContractTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建带内存文件存储行为的批量导入测试夹具。
|
||||
*
|
||||
* @return 批量导入测试夹具
|
||||
*/
|
||||
private BatchFixture batchImportService() {
|
||||
SkillService skillService = mock(SkillService.class);
|
||||
when(skillService.list(any(QueryWrapper.class))).thenReturn(List.of());
|
||||
FileStorageService fileStorageService = mock(FileStorageService.class);
|
||||
SkillImportStageStore stageStore = mock(SkillImportStageStore.class);
|
||||
Map<String, byte[]> storedFiles = new HashMap<>();
|
||||
AtomicInteger sequence = new AtomicInteger();
|
||||
try {
|
||||
when(fileStorageService.save(any(java.io.File.class), anyString())).thenAnswer(invocation -> {
|
||||
java.io.File file = invocation.getArgument(0);
|
||||
String path = "stored-" + sequence.incrementAndGet() + ".zip";
|
||||
storedFiles.put(path, Files.readAllBytes(file.toPath()));
|
||||
return path;
|
||||
});
|
||||
when(fileStorageService.readStream(anyString())).thenAnswer(invocation ->
|
||||
new ByteArrayInputStream(storedFiles.get(invocation.getArgument(0))));
|
||||
} catch (IOException exception) {
|
||||
throw new IllegalStateException("创建测试文件存储失败", exception);
|
||||
}
|
||||
when(stageStore.create(anyString(), anyString())).thenAnswer(invocation -> {
|
||||
SkillImportStage stage = new SkillImportStage();
|
||||
stage.setFilePath(invocation.getArgument(0));
|
||||
stage.setOriginalName(invocation.getArgument(1));
|
||||
stage.setImportToken("token-" + sequence.get());
|
||||
stage.setExpiresAt(new Date(System.currentTimeMillis() + 60_000));
|
||||
return stage;
|
||||
});
|
||||
SkillImportServiceImpl service = new SkillImportServiceImpl(skillService,
|
||||
mock(DBSkillContentStore.class), fileStorageService, stageStore,
|
||||
mock(ResourceAccessService.class));
|
||||
return new BatchFixture(service, fileStorageService, stageStore);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建可重复打开输入流的上传文件替身。
|
||||
*
|
||||
* @param name 原始文件名
|
||||
* @param bytes 文件字节
|
||||
* @return 上传文件替身
|
||||
*/
|
||||
private MultipartFile multipartFile(String name, byte[] bytes) {
|
||||
MultipartFile file = mock(MultipartFile.class);
|
||||
when(file.isEmpty()).thenReturn(false);
|
||||
when(file.getSize()).thenReturn((long) bytes.length);
|
||||
when(file.getOriginalFilename()).thenReturn(name);
|
||||
try {
|
||||
when(file.getInputStream()).thenAnswer(ignored -> new ByteArrayInputStream(bytes));
|
||||
} catch (IOException exception) {
|
||||
throw new IllegalStateException("创建测试上传文件失败", exception);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private void write(ZipOutputStream zip, String path, String content) throws Exception {
|
||||
zip.putNextEntry(new ZipEntry(path));
|
||||
zip.write(content.getBytes(StandardCharsets.UTF_8));
|
||||
@@ -189,4 +310,16 @@ public class StandardSkillPackageContractTest {
|
||||
throw new IllegalStateException("读取标准 Skill 测试包失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量预检测试依赖集合。
|
||||
*
|
||||
* @param service 导入服务
|
||||
* @param fileStorageService 文件存储
|
||||
* @param stageStore 导入暂存仓库
|
||||
*/
|
||||
private record BatchFixture(SkillImportServiceImpl service,
|
||||
FileStorageService fileStorageService,
|
||||
SkillImportStageStore stageStore) {
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user