diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/SkillController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/SkillController.java index 4ca3695c..f254bbcf 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/SkillController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/SkillController.java @@ -16,6 +16,8 @@ import org.springframework.web.multipart.MultipartFile; import tech.easyflow.admin.controller.ai.support.AiResourceCreatorNameSupport; import tech.easyflow.admin.controller.skill.vo.SkillCopyRequest; import tech.easyflow.admin.controller.skill.vo.SkillDraftRequest; +import tech.easyflow.admin.controller.skill.vo.SkillGitRepositoryPrepareRequest; +import tech.easyflow.admin.controller.skill.vo.SkillGitRepositoryScanRequest; import tech.easyflow.admin.controller.skill.vo.SkillImportBatchResultView; import tech.easyflow.admin.controller.skill.vo.SkillView; import tech.easyflow.admin.controller.skill.vo.SkillToolBindingUpdateRequest; @@ -39,6 +41,8 @@ import tech.easyflow.skill.imports.SkillExportService; import tech.easyflow.skill.imports.SkillImportConfirmRequest; import tech.easyflow.skill.imports.SkillImportPreview; import tech.easyflow.skill.imports.SkillImportService; +import tech.easyflow.skill.gitimport.SkillGitImportService; +import tech.easyflow.skill.gitimport.SkillGitScanResult; import tech.easyflow.skill.publish.SkillPublishAppService; import tech.easyflow.skill.security.SkillVisibilityQueryHelper; import tech.easyflow.skill.service.SkillApprovalStateService; @@ -77,6 +81,7 @@ public class SkillController { private final SkillApprovalStateService skillApprovalStateService; private final SkillPublishAppService skillPublishAppService; private final SkillImportService skillImportService; + private final SkillGitImportService skillGitImportService; private final SkillExportService skillExportService; private final SkillFileService skillFileService; private final SkillToolBindingService skillToolBindingService; @@ -93,6 +98,7 @@ public class SkillController { * @param skillApprovalStateService 审批状态服务 * @param skillPublishAppService 发布服务 * @param skillImportService 导入服务 + * @param skillGitImportService Git 仓库导入服务 * @param skillExportService 导出服务 * @param skillFileService 文件服务 * @param skillToolBindingService Skill Tool 绑定服务 @@ -106,6 +112,7 @@ public class SkillController { SkillApprovalStateService skillApprovalStateService, SkillPublishAppService skillPublishAppService, SkillImportService skillImportService, + SkillGitImportService skillGitImportService, SkillExportService skillExportService, SkillFileService skillFileService, SkillToolBindingService skillToolBindingService, @@ -118,6 +125,7 @@ public class SkillController { this.skillApprovalStateService = skillApprovalStateService; this.skillPublishAppService = skillPublishAppService; this.skillImportService = skillImportService; + this.skillGitImportService = skillGitImportService; this.skillExportService = skillExportService; this.skillFileService = skillFileService; this.skillToolBindingService = skillToolBindingService; @@ -447,10 +455,43 @@ public class SkillController { if (uploads.isEmpty()) { throw new BusinessException("请选择要导入的标准 Skill ZIP"); } - if (uploads.size() > 20) { - throw new BusinessException("单次最多预检 20 个 Skill ZIP"); + if (uploads.size() > SkillImportService.MAX_BATCH_SKILL_COUNT) { + throw new BusinessException("单次最多选择 " + + SkillImportService.MAX_BATCH_SKILL_COUNT + " 个 Skill ZIP"); } - return Result.ok(uploads.stream().map(skillImportService::preview).toList()); + return Result.ok(skillImportService.previewBatch(uploads)); + } + + /** + * 扫描 HTTPS Git 仓库中的标准 Skill 候选。 + * + * @param request 仓库地址请求 + * @return 固定提交的候选列表 + */ + @PostMapping("/import/repository/scan") + @SaCheckPermission("/api/v1/skill/import") + public Result scanGitRepository( + @JsonBody(required = true, skipConvertError = false) SkillGitRepositoryScanRequest request) { + if (request == null) { + throw new BusinessException("请输入 Git 仓库地址"); + } + return Result.ok(skillGitImportService.scan(request.repositoryUrl())); + } + + /** + * 将选中的 Git Skill 候选转换为既有标准 ZIP 导入预览。 + * + * @param request 扫描令牌与候选 ID + * @return 与本地 ZIP 导入一致的预览列表 + */ + @PostMapping("/import/repository/prepare") + @SaCheckPermission("/api/v1/skill/import") + public Result> prepareGitRepositoryImport( + @JsonBody(required = true, skipConvertError = false) SkillGitRepositoryPrepareRequest request) { + if (request == null) { + throw new BusinessException("请选择要导入的 Git Skill"); + } + return Result.ok(skillGitImportService.prepare(request.scanToken(), request.candidateIds())); } /** @@ -479,8 +520,9 @@ public class SkillController { if (requests == null || requests.isEmpty()) { throw new BusinessException("请选择要确认导入的 Skill"); } - if (requests.size() > 20) { - throw new BusinessException("单次最多确认导入 20 个 Skill"); + if (requests.size() > SkillImportService.MAX_BATCH_SKILL_COUNT) { + throw new BusinessException("单次最多确认导入 " + + SkillImportService.MAX_BATCH_SKILL_COUNT + " 个 Skill"); } List results = new java.util.ArrayList<>(requests.size()); for (SkillImportConfirmRequest request : requests) { diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/vo/SkillGitRepositoryPrepareRequest.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/vo/SkillGitRepositoryPrepareRequest.java new file mode 100644 index 00000000..3e81b60d --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/vo/SkillGitRepositoryPrepareRequest.java @@ -0,0 +1,12 @@ +package tech.easyflow.admin.controller.skill.vo; + +import java.util.List; + +/** + * 选中 Git Skill 候选的标准导入预览准备请求。 + * + * @param scanToken 短期扫描令牌 + * @param candidateIds 选中的候选 ID + */ +public record SkillGitRepositoryPrepareRequest(String scanToken, List candidateIds) { +} diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/vo/SkillGitRepositoryScanRequest.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/vo/SkillGitRepositoryScanRequest.java new file mode 100644 index 00000000..96440a84 --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/skill/vo/SkillGitRepositoryScanRequest.java @@ -0,0 +1,9 @@ +package tech.easyflow.admin.controller.skill.vo; + +/** + * Git 仓库 Skill 扫描请求。 + * + * @param repositoryUrl HTTPS Git 仓库地址，可省略 .git 后缀 + */ +public record SkillGitRepositoryScanRequest(String repositoryUrl) { +} diff --git a/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/skill/SkillControllerContractTest.java b/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/skill/SkillControllerContractTest.java index 3d27408b..5cdec9fb 100644 --- a/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/skill/SkillControllerContractTest.java +++ b/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/skill/SkillControllerContractTest.java @@ -8,6 +8,8 @@ import org.testng.annotations.Test; import tech.easyflow.admin.controller.ai.support.AiResourceCreatorNameSupport; import tech.easyflow.admin.controller.skill.vo.SkillCopyRequest; import tech.easyflow.admin.controller.skill.vo.SkillDraftRequest; +import tech.easyflow.admin.controller.skill.vo.SkillGitRepositoryPrepareRequest; +import tech.easyflow.admin.controller.skill.vo.SkillGitRepositoryScanRequest; import tech.easyflow.admin.controller.skill.vo.SkillImportBatchResultView; import tech.easyflow.admin.controller.skill.vo.SkillToolBindingUpdateRequest; import tech.easyflow.admin.controller.skill.vo.SkillView; @@ -19,6 +21,7 @@ import tech.easyflow.common.web.jsonbody.JsonBody; import tech.easyflow.common.web.jsonbody.JsonBodyParser; import tech.easyflow.skill.entity.Skill; import tech.easyflow.skill.file.SkillFileService; +import tech.easyflow.skill.gitimport.SkillGitImportService; import tech.easyflow.skill.imports.SkillExportService; import tech.easyflow.skill.imports.SkillImportConfirmRequest; import tech.easyflow.skill.imports.SkillImportService; @@ -144,6 +147,26 @@ public class SkillControllerContractTest { .anyMatch("/api/v1/skill/capability"::equals)); } + /** + * Git 仓库扫描与候选准备沿用 Skill 导入权限，并使用白名单请求 DTO。 + * + * @throws Exception 反射失败 + */ + @Test + public void gitRepositoryEndpointsReuseImportPermission() throws Exception { + Method scan = SkillController.class.getMethod( + "scanGitRepository", SkillGitRepositoryScanRequest.class); + Method prepare = SkillController.class.getMethod( + "prepareGitRepositoryImport", SkillGitRepositoryPrepareRequest.class); + + Assert.assertEquals(scan.getAnnotation(SaCheckPermission.class).value(), + new String[]{"/api/v1/skill/import"}); + Assert.assertEquals(prepare.getAnnotation(SaCheckPermission.class).value(), + new String[]{"/api/v1/skill/import"}); + Assert.assertNotNull(scan.getParameters()[0].getAnnotation(JsonBody.class)); + Assert.assertNotNull(prepare.getParameters()[0].getAnnotation(JsonBody.class)); + } + /** * 发布入口把必填发布说明原样交给应用服务。 */ @@ -195,7 +218,8 @@ public class SkillControllerContractTest { ResourceAccessService accessService = mock(ResourceAccessService.class); when(accessService.canAccess(any(), any(), any())).thenReturn(true); return new SkillController(mock(SkillService.class), mock(SkillApprovalStateService.class), - publishService, importService, mock(SkillExportService.class), mock(SkillFileService.class), + publishService, importService, mock(SkillGitImportService.class), + mock(SkillExportService.class), mock(SkillFileService.class), mock(SkillToolBindingService.class), mock(SkillToolOptionQueryService.class), accessService, mock(CategoryPermissionService.class), mock(SkillVisibilityQueryHelper.class), mock(AiResourceCreatorNameSupport.class)); diff --git a/easyflow-modules/easyflow-module-skill/pom.xml b/easyflow-modules/easyflow-module-skill/pom.xml index 1407c2eb..1d860453 100644 --- a/easyflow-modules/easyflow-module-skill/pom.xml +++ b/easyflow-modules/easyflow-module-skill/pom.xml @@ -61,6 +61,16 @@ org.apache.commons commons-compress + + org.eclipse.jgit + org.eclipse.jgit + 7.7.0.202606012155-r + + + org.eclipse.jgit + org.eclipse.jgit.http.apache + 7.7.0.202606012155-r + junit junit diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/GitRepositoryAccessPolicy.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/GitRepositoryAccessPolicy.java new file mode 100644 index 00000000..f4e5b618 --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/GitRepositoryAccessPolicy.java @@ -0,0 +1,183 @@ +package tech.easyflow.skill.gitimport; + +import org.springframework.stereotype.Component; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.net.URL; +import java.util.Arrays; +import java.util.Locale; + +/** + * Git 仓库远程地址访问策略，负责协议、凭证、端口与 SSRF 边界校验。 + */ +@Component +public class GitRepositoryAccessPolicy { + + private final SkillGitImportProperties properties; + private final HostResolver hostResolver; + + /** + * 创建 Git 地址访问策略。 + * + * @param properties Git 导入配置 + */ + public GitRepositoryAccessPolicy(SkillGitImportProperties properties) { + this(properties, InetAddress::getAllByName); + } + + GitRepositoryAccessPolicy(SkillGitImportProperties properties, HostResolver hostResolver) { + this.properties = properties; + this.hostResolver = hostResolver; + } + + /** + * 校验并规范化用户输入的 Git 仓库地址。 + * + * @param repositoryUrl 用户输入地址 + * @return 规范化 HTTPS 地址 + * @throws BusinessException 地址不符合安全边界 + */ + public String normalizeRepositoryUrl(String repositoryUrl) { + if (repositoryUrl == null || repositoryUrl.isBlank()) { + throw new BusinessException("请输入 Git 仓库地址"); + } + URI uri; + try { + uri = new URI(repositoryUrl.trim()).normalize(); + } catch (URISyntaxException exception) { + throw new BusinessException("Git 仓库地址格式不正确"); + } + validateUri(uri, true); + String path = uri.getRawPath(); + if (path == null || path.isBlank() || "/".equals(path)) { + throw new BusinessException("Git 仓库地址缺少仓库路径"); + } + String normalizedPath = path.length() > 1 && path.endsWith("/") + ? path.substring(0, path.length() - 1) : path; + try { + return new URI("https", null, uri.getHost().toLowerCase(Locale.ROOT), + uri.getPort(), normalizedPath, null, null).toASCIIString(); + } catch (URISyntaxException exception) { + throw new BusinessException("Git 仓库地址格式不正确"); + } + } + + /** + * 校验 JGit 实际发起的连接地址，包括重定向后的服务地址。 + * + * @param url 连接地址 + * @throws java.io.IOException 地址被访问策略拒绝 + */ + ResolvedConnection resolveConnection(URL url) throws IOException { + try { + URI uri = url.toURI(); + InetAddress[] addresses = validateUri(uri, false); + return new ResolvedConnection(uri.getHost().toLowerCase(Locale.ROOT), addresses); + } catch (URISyntaxException | BusinessException exception) { + throw new IOException(exception.getMessage(), exception); + } + } + + private InetAddress[] validateUri(URI uri, boolean repositoryInput) { + if (!"https".equalsIgnoreCase(uri.getScheme())) { + throw new BusinessException("Git 仓库仅支持 HTTPS 地址"); + } + if (uri.getUserInfo() != null) { + throw new BusinessException("Git 仓库地址不能包含用户名或密码"); + } + if (repositoryInput && (uri.getQuery() != null || uri.getFragment() != null)) { + throw new BusinessException("Git 仓库地址不能包含查询参数或片段"); + } + String host = uri.getHost(); + if (host == null || host.isBlank()) { + throw new BusinessException("Git 仓库地址缺少有效主机"); + } + int port = uri.getPort() < 0 ? 443 : uri.getPort(); + if (!properties.getAllowedPorts().contains(port)) { + throw new BusinessException("Git 仓库端口不在允许范围内"); + } + String normalizedHost = host.toLowerCase(Locale.ROOT); + InetAddress[] addresses; + try { + addresses = hostResolver.resolve(normalizedHost); + } catch (UnknownHostException exception) { + throw new BusinessException("无法解析 Git 仓库主机"); + } + if (addresses.length == 0) { + throw new BusinessException("无法解析 Git 仓库主机"); + } + if (!properties.getTrustedPrivateHosts().contains(normalizedHost) + && Arrays.stream(addresses).anyMatch(this::isBlockedAddress)) { + throw new BusinessException("Git 仓库地址指向受限网络"); + } + return addresses.clone(); + } + + private boolean isBlockedAddress(InetAddress address) { + if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() + || address.isSiteLocalAddress() || address.isMulticastAddress()) { + return true; + } + byte[] bytes = address.getAddress(); + if (address instanceof Inet4Address && bytes.length == 4) { + int first = Byte.toUnsignedInt(bytes[0]); + int second = Byte.toUnsignedInt(bytes[1]); + int third = Byte.toUnsignedInt(bytes[2]); + return first == 0 || first == 10 || first == 127 || first >= 224 + || (first == 100 && second >= 64 && second <= 127) + || (first == 169 && second == 254) + || (first == 172 && second >= 16 && second <= 31) + || (first == 192 && second == 0 && (third == 0 || third == 2)) + || (first == 192 && second == 88 && third == 99) + || (first == 192 && second == 168) + || (first == 198 && (second == 18 || second == 19)) + || (first == 198 && second == 51 && third == 100) + || (first == 203 && second == 0 && third == 113); + } + if (address instanceof Inet6Address && bytes.length == 16) { + int first = Byte.toUnsignedInt(bytes[0]); + int second = Byte.toUnsignedInt(bytes[1]); + return (first & 0xFE) == 0xFC || (first == 0xFE && (second & 0xC0) == 0x80) + || (first == 0x20 && second == 0x01 && Byte.toUnsignedInt(bytes[2]) == 0x0D + && Byte.toUnsignedInt(bytes[3]) == 0xB8); + } + return false; + } + + @FunctionalInterface + interface HostResolver { + + /** + * 解析目标主机的全部地址。 + * + * @param host 规范化主机名 + * @return 解析地址 + * @throws UnknownHostException 主机无法解析 + */ + InetAddress[] resolve(String host) throws UnknownHostException; + } + + record ResolvedConnection(String host, InetAddress[] addresses) { + + ResolvedConnection { + addresses = addresses.clone(); + } + + /** + * 获取用于实际连接的固定地址。 + * + * @return 地址副本 + */ + @Override + public InetAddress[] addresses() { + return addresses.clone(); + } + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitCandidateScanner.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitCandidateScanner.java new file mode 100644 index 00000000..8295c69e --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitCandidateScanner.java @@ -0,0 +1,416 @@ +package tech.easyflow.skill.gitimport; + +import com.easyagents.skill.exception.SkillValidationException; +import com.easyagents.skill.factory.SkillFactory; +import com.easyagents.skill.model.Skill; +import com.easyagents.skill.model.SkillPackageLimits; +import com.easyagents.skill.util.SkillHashes; +import com.easyagents.skill.util.SkillPaths; +import com.easyagents.skill.util.SkillResources; +import com.easyagents.skill.validation.SkillValidationIssue; +import com.easyagents.skill.validation.SkillValidationSeverity; +import com.easyagents.skill.validation.defaults.DefaultSkillValidator; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.IOException; +import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * 固定 Git Tree 中的标准 Skill 候选发现、校验与标准包组装器。 + */ +@Component +public class SkillGitCandidateScanner { + + private static final Logger LOG = LoggerFactory.getLogger(SkillGitCandidateScanner.class); + private static final int LFS_POINTER_MAX_BYTES = 1_024; + private static final byte[] LFS_HEADER = "version https://git-lfs.github.com/spec/v1" + .getBytes(StandardCharsets.UTF_8); + + private final SkillGitImportProperties properties; + private final SkillPackageLimits packageLimits = SkillPackageLimits.defaults(); + private final DefaultSkillValidator validator = new DefaultSkillValidator(); + + /** + * 创建候选扫描器。 + * + * @param properties Git 导入配置 + */ + public SkillGitCandidateScanner(SkillGitImportProperties properties) { + this.properties = properties; + } + + /** + * 扫描快照固定提交中的全部标准 Skill 候选。 + * + * @param snapshot 固定提交快照 + * @return 候选及其资源归属 + * @throws BusinessException 仓库结构不可读取或超过安全限额 + */ + public SkillGitRepositoryScan scan(SkillGitRepositoryReader.Snapshot snapshot) { + Repository repository = snapshot.repository(); + try (RevWalk revWalk = new RevWalk(repository)) { + RevCommit commit = revWalk.parseCommit(ObjectId.fromString(snapshot.commitSha())); + List entries = readEntries(repository, commit); + Set roots = discoverCandidateRoots(entries); + if (roots.size() > properties.getMaxCandidates()) { + throw new BusinessException(413, 4133, "Git 仓库中的 Skill 候选数量超过限制"); + } + Map> ownedEntries = assignEntries(entries, roots); + List candidates = new ArrayList<>(roots.size()); + Map> filesByCandidate = new LinkedHashMap<>(); + roots.stream().sorted().forEach(root -> { + CandidateBuild build = buildCandidate(repository, snapshot.commitSha(), root, + ownedEntries.getOrDefault(root, List.of())); + candidates.add(build.candidate()); + filesByCandidate.put(build.candidate().candidateId(), build.files()); + }); + return new SkillGitRepositoryScan(candidates, filesByCandidate); + } catch (IOException exception) { + throw new BusinessException(500, 500, "读取 Git 仓库文件树失败", exception); + } + } + + /** + * 将一个可导入候选组装为包含单个标准 Skill 的 ZIP。 + * + * @param snapshot 固定提交快照 + * @param scan 同一快照的扫描数据 + * @param candidateId 候选 ID + * @return 待交给标准包预检的临时 ZIP + * @throws BusinessException 候选不存在、不可导入或文件读取失败 + */ + public Path createStandardZip(SkillGitRepositoryReader.Snapshot snapshot, + SkillGitRepositoryScan scan, + String candidateId) { + SkillGitScanResult.Candidate candidate = scan.candidates().stream() + .filter(item -> item.candidateId().equals(candidateId)) + .findFirst() + .orElseThrow(() -> new BusinessException("Git Skill 候选不存在，请重新扫描")); + if (!candidate.importable()) { + throw new BusinessException("Git Skill 候选存在校验问题，不能导入：" + candidate.path()); + } + List files = scan.filesByCandidate().get(candidateId); + if (files == null || files.stream().noneMatch(file -> SkillPaths.SKILL_FILE.equals(file.relativePath()))) { + throw new BusinessException("Git Skill 候选缺少 SKILL.md，请重新扫描"); + } + Path archive; + try { + archive = Files.createTempFile("easyflow-git-skill-", ".zip"); + } catch (IOException exception) { + throw new BusinessException(500, 500, "创建 Git Skill 临时包失败", exception); + } + try (ZipOutputStream output = new ZipOutputStream(Files.newOutputStream(archive))) { + for (SkillGitRepositoryScan.RepositoryFile file : files.stream() + .sorted(Comparator.comparing(SkillGitRepositoryScan.RepositoryFile::relativePath)).toList()) { + FileMode mode = FileMode.fromBits(file.fileModeBits()); + if (!isRegular(mode)) { + throw new BusinessException("Git Skill 包含不支持的文件对象：" + file.relativePath()); + } + ZipEntry entry = new ZipEntry(candidate.name() + "/" + file.relativePath()); + entry.setTime(0L); + output.putNextEntry(entry); + snapshot.repository().open(file.objectId(), Constants.OBJ_BLOB).copyTo(output); + output.closeEntry(); + } + return archive; + } catch (IOException | RuntimeException exception) { + deleteArchive(archive); + if (exception instanceof BusinessException businessException) { + throw businessException; + } + throw new BusinessException(500, 500, "组装 Git Skill 标准包失败", exception); + } + } + + /** + * 删除扫描器创建的临时标准包。 + * + * @param archive 临时 ZIP + */ + public void deleteArchive(Path archive) { + if (archive == null) { + return; + } + try { + Files.deleteIfExists(archive); + } catch (IOException exception) { + // 临时包删除失败不能覆盖已生成的导入预览；记录完整路径供运维清理。 + LOG.warn("清理 Git Skill 临时包失败，path={}", archive, exception); + } + } + + private List readEntries(Repository repository, RevCommit commit) throws IOException { + List entries = new ArrayList<>(); + try (TreeWalk treeWalk = new TreeWalk(repository)) { + treeWalk.addTree(commit.getTree()); + treeWalk.setRecursive(true); + while (treeWalk.next()) { + if (entries.size() >= properties.getMaxRepositoryFiles()) { + throw new BusinessException(413, 4134, "Git 仓库文件数量超过限制"); + } + FileMode mode = treeWalk.getFileMode(0); + ObjectId objectId = treeWalk.getObjectId(0).copy(); + long size = isRegular(mode) || FileMode.SYMLINK.equals(mode) + ? repository.open(objectId, Constants.OBJ_BLOB).getSize() : 0L; + entries.add(new RawEntry(treeWalk.getPathString(), objectId, mode, size)); + } + } + return entries; + } + + private Set discoverCandidateRoots(List entries) { + Set roots = new LinkedHashSet<>(); + for (RawEntry entry : entries) { + if (isRegular(entry.mode()) && SkillPaths.SKILL_FILE.equals(fileName(entry.path()))) { + roots.add(parentPath(entry.path())); + } + } + return roots; + } + + private Map> assignEntries(List entries, Set roots) { + Map> result = new HashMap<>(); + roots.forEach(root -> result.put(root, new ArrayList<>())); + for (RawEntry entry : entries) { + String owner = nearestCandidateRoot(entry.path(), roots); + if (owner != null) { + result.get(owner).add(entry); + } + } + return result; + } + + private String nearestCandidateRoot(String filePath, Set roots) { + String parent = parentPath(filePath); + while (parent != null) { + if (roots.contains(parent)) { + return parent; + } + parent = parent.isEmpty() ? null : parentPath(parent); + } + return null; + } + + private CandidateBuild buildCandidate(Repository repository, String commitSha, String root, + List ownedEntries) { + RawEntry skillFile = ownedEntries.stream() + .filter(entry -> isRegular(entry.mode())) + .filter(entry -> SkillPaths.SKILL_FILE.equals(relativePath(root, entry.path()))) + .findFirst().orElseThrow(() -> new BusinessException("Git Skill 候选缺少 SKILL.md")); + List issues = new ArrayList<>(); + String fallbackName = root.isEmpty() ? "root-skill" : fileName(root); + String name = fallbackName; + String description = ""; + String skillContent = null; + if (skillFile.size() > packageLimits.getMaxTextFileBytes()) { + issues.add(issue("TEXT_FILE_SIZE_LIMIT", "SKILL.md 超过允许大小", SkillPaths.SKILL_FILE)); + } else { + try { + skillContent = decodeUtf8(loadBytes(repository, skillFile)); + Skill skill = SkillFactory.create(skillContent); + skill.setPackageRoot(root.isEmpty() ? skill.getName() : fileName(root)); + name = skill.getName(); + description = skill.getDescription(); + validator.validateReport(skill, packageLimits).getIssues().stream() + .filter(item -> item.getSeverity() == SkillValidationSeverity.ERROR) + .map(this::toIssue) + .forEach(issues::add); + } catch (SkillValidationException exception) { + issues.add(issue(exception.getCode() == null ? "INVALID_SKILL" : exception.getCode(), + exception.getMessage(), SkillPaths.SKILL_FILE)); + } catch (CharacterCodingException exception) { + issues.add(issue("INVALID_UTF8", "SKILL.md 不是有效的 UTF-8 文本", SkillPaths.SKILL_FILE)); + } catch (IOException exception) { + throw new BusinessException(500, 500, "读取 Git SKILL.md 失败", exception); + } + } + + List files = new ArrayList<>(ownedEntries.size()); + Set collisionKeys = new HashSet<>(); + collisionKeys.add(SkillPaths.collisionKey(SkillPaths.SKILL_FILE)); + long totalBytes = 0L; + int resourceCount = 0; + for (RawEntry entry : ownedEntries) { + String relative = relativePath(root, entry.path()); + totalBytes = addSize(totalBytes, entry.size()); + files.add(new SkillGitRepositoryScan.RepositoryFile(entry.path(), relative, + entry.objectId(), entry.size(), entry.mode().getBits())); + if (SkillPaths.SKILL_FILE.equals(relative)) { + continue; + } + resourceCount++; + validateResource(repository, entry, relative, collisionKeys, issues); + } + if (resourceCount + 1 > packageLimits.getMaxEntryCount()) { + issues.add(issue("ENTRY_COUNT_LIMIT", "Skill 文件数量超过允许上限", null)); + } + if (totalBytes > packageLimits.getMaxTotalUncompressedBytes()) { + issues.add(issue("TOTAL_SIZE_LIMIT", "Skill 资源总大小超过允许上限", null)); + } + String candidateId = SkillHashes.sha256Hex((commitSha + "\n" + root + "\n" + + skillFile.objectId().name()).getBytes(StandardCharsets.UTF_8)); + SkillGitScanResult.CandidateStatus status = issues.isEmpty() + ? SkillGitScanResult.CandidateStatus.IMPORTABLE + : SkillGitScanResult.CandidateStatus.NEEDS_ATTENTION; + SkillGitScanResult.Candidate candidate = new SkillGitScanResult.Candidate( + candidateId, root.isEmpty() ? "/" : root, name, description, status, + resourceCount, totalBytes, issues); + return new CandidateBuild(candidate, files); + } + + private void validateResource(Repository repository, RawEntry entry, String relative, + Set collisionKeys, List issues) { + if (FileMode.SYMLINK.equals(entry.mode())) { + issues.add(issue("GIT_SYMLINK_UNSUPPORTED", "暂不支持符号链接", relative)); + return; + } + if (FileMode.GITLINK.equals(entry.mode())) { + issues.add(issue("GIT_SUBMODULE_UNSUPPORTED", "暂不支持 Git 子模块", relative)); + return; + } + if (!isRegular(entry.mode())) { + issues.add(issue("GIT_OBJECT_UNSUPPORTED", "不支持该 Git 文件对象类型", relative)); + return; + } + String normalized; + try { + normalized = SkillPaths.normalize(relative); + } catch (SkillValidationException exception) { + issues.add(issue("UNSAFE_RESOURCE_PATH", "资源路径不符合 Skill 规范", relative)); + return; + } + String collisionKey = SkillPaths.collisionKey(normalized); + if (!collisionKeys.add(collisionKey)) { + issues.add(issue("DUPLICATE_RESOURCE_PATH", "资源路径存在大小写或 Unicode 冲突", relative)); + } + if (normalized.length() > packageLimits.getMaxPathLength() + || SkillPaths.depth(normalized) + 1 > packageLimits.getMaxPathDepth()) { + issues.add(issue("RESOURCE_PATH_LIMIT", "资源路径长度或深度超过限制", relative)); + } + String mediaType = mediaType(relative); + boolean text = SkillResources.isText(relative, mediaType); + long fileLimit = text ? packageLimits.getMaxTextFileBytes() : packageLimits.getMaxBinaryFileBytes(); + if (entry.size() > fileLimit) { + issues.add(issue(text ? "TEXT_FILE_SIZE_LIMIT" : "BINARY_FILE_SIZE_LIMIT", + "资源文件超过允许大小", relative)); + return; + } + try { + byte[] bytes = null; + if (text) { + bytes = loadBytes(repository, entry); + decodeUtf8(bytes); + } + if (entry.size() <= LFS_POINTER_MAX_BYTES + && isLfsPointer(bytes == null ? loadBytes(repository, entry) : bytes)) { + issues.add(issue("GIT_LFS_UNRESOLVED", "Git LFS 文件尚未包含真实内容", relative)); + } + } catch (CharacterCodingException exception) { + issues.add(issue("INVALID_UTF8", "文本资源不是有效的 UTF-8 内容", relative)); + } catch (IOException exception) { + throw new BusinessException(500, 500, "读取 Git Skill 资源失败：" + relative, exception); + } + } + + private byte[] loadBytes(Repository repository, RawEntry entry) throws IOException { + ObjectLoader loader = repository.open(entry.objectId(), Constants.OBJ_BLOB); + long maximum = Math.max(packageLimits.getMaxBinaryFileBytes(), packageLimits.getMaxTextFileBytes()); + if (loader.getSize() > maximum) { + throw new IOException("Git Blob exceeds the configured single-file limit"); + } + return loader.getBytes((int) Math.min(Integer.MAX_VALUE, maximum)); + } + + private String decodeUtf8(byte[] bytes) throws CharacterCodingException { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } + + private boolean isLfsPointer(byte[] bytes) { + if (bytes.length < LFS_HEADER.length) { + return false; + } + for (int index = 0; index < LFS_HEADER.length; index++) { + if (bytes[index] != LFS_HEADER[index]) { + return false; + } + } + return true; + } + + private SkillGitScanResult.Issue toIssue(SkillValidationIssue source) { + return issue(source.getCode(), source.getMessage(), source.getPath()); + } + + private SkillGitScanResult.Issue issue(String code, String message, String path) { + return new SkillGitScanResult.Issue(code, message, path); + } + + private String mediaType(String path) { + String guessed = URLConnection.guessContentTypeFromName(path); + return guessed == null ? "application/octet-stream" : guessed; + } + + private long addSize(long total, long size) { + try { + return Math.addExact(total, Math.max(0L, size)); + } catch (ArithmeticException exception) { + return Long.MAX_VALUE; + } + } + + private boolean isRegular(FileMode mode) { + return FileMode.REGULAR_FILE.equals(mode) || FileMode.EXECUTABLE_FILE.equals(mode); + } + + private String relativePath(String root, String path) { + return root.isEmpty() ? path : path.substring(root.length() + 1); + } + + private String parentPath(String path) { + int index = path.lastIndexOf('/'); + return index < 0 ? "" : path.substring(0, index); + } + + private String fileName(String path) { + int index = path.lastIndexOf('/'); + return index < 0 ? path : path.substring(index + 1); + } + + private record RawEntry(String path, ObjectId objectId, FileMode mode, long size) { + } + + private record CandidateBuild(SkillGitScanResult.Candidate candidate, + List files) { + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitImportProperties.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitImportProperties.java new file mode 100644 index 00000000..61c1db5b --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitImportProperties.java @@ -0,0 +1,258 @@ +package tech.easyflow.skill.gitimport; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.time.Duration; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Skill Git 仓库导入的网络与资源边界配置。 + */ +@Configuration +@ConfigurationProperties(prefix = "easyflow.skill.git-import") +public class SkillGitImportProperties { + + /** Git 网络操作超时时间。 */ + private Duration timeout = Duration.ofSeconds(45); + + /** 单次 Git 远程操作总超时时间。 */ + private Duration operationTimeout = Duration.ofSeconds(90); + + /** 单实例允许同时执行的 Git 远程操作数量。 */ + private int maxConcurrentOperations = 4; + + /** 同一账号两次仓库扫描之间的最小间隔。 */ + private Duration scanMinimumInterval = Duration.ofSeconds(2); + + /** 扫描令牌有效期。 */ + private Duration scanTtl = Duration.ofMinutes(30); + + /** 临时裸仓库最大磁盘占用。 */ + private long maxRepositoryBytes = 256L * 1024 * 1024; + + /** 单仓库最大文件对象数量。 */ + private int maxRepositoryFiles = 50_000; + + /** 单仓库最大 Skill 候选数量。 */ + private int maxCandidates = 2_000; + + /** 允许访问的 HTTPS 端口。 */ + private Set allowedPorts = new LinkedHashSet<>(Set.of(443)); + + /** 允许解析到私网地址的可信 Git 主机。 */ + private Set trustedPrivateHosts = new LinkedHashSet<>(); + + /** + * 获取 Git 网络操作超时时间。 + * + * @return 超时时间 + */ + public Duration getTimeout() { + return timeout; + } + + /** + * 设置 Git 网络操作超时时间。 + * + * @param timeout 超时时间 + */ + public void setTimeout(Duration timeout) { + this.timeout = timeout == null ? Duration.ofSeconds(45) : timeout; + } + + /** + * 获取单次 Git 远程操作总超时时间。 + * + * @return 总超时时间 + */ + public Duration getOperationTimeout() { + return operationTimeout; + } + + /** + * 设置单次 Git 远程操作总超时时间。 + * + * @param operationTimeout 总超时时间 + */ + public void setOperationTimeout(Duration operationTimeout) { + this.operationTimeout = positive(operationTimeout, "operationTimeout"); + } + + /** + * 获取单实例 Git 远程操作并发上限。 + * + * @return 并发上限 + */ + public int getMaxConcurrentOperations() { + return maxConcurrentOperations; + } + + /** + * 设置单实例 Git 远程操作并发上限。 + * + * @param maxConcurrentOperations 并发上限 + */ + public void setMaxConcurrentOperations(int maxConcurrentOperations) { + this.maxConcurrentOperations = positive(maxConcurrentOperations, "maxConcurrentOperations"); + } + + /** + * 获取同一账号两次仓库扫描的最小间隔。 + * + * @return 最小间隔 + */ + public Duration getScanMinimumInterval() { + return scanMinimumInterval; + } + + /** + * 设置同一账号两次仓库扫描的最小间隔。 + * + * @param scanMinimumInterval 最小间隔 + */ + public void setScanMinimumInterval(Duration scanMinimumInterval) { + this.scanMinimumInterval = positive(scanMinimumInterval, "scanMinimumInterval"); + } + + /** + * 获取扫描令牌有效期。 + * + * @return 有效期 + */ + public Duration getScanTtl() { + return scanTtl; + } + + /** + * 设置扫描令牌有效期。 + * + * @param scanTtl 有效期 + */ + public void setScanTtl(Duration scanTtl) { + this.scanTtl = scanTtl == null ? Duration.ofMinutes(30) : scanTtl; + } + + /** + * 获取临时仓库大小上限。 + * + * @return 字节数 + */ + public long getMaxRepositoryBytes() { + return maxRepositoryBytes; + } + + /** + * 设置临时仓库大小上限。 + * + * @param maxRepositoryBytes 字节数 + */ + public void setMaxRepositoryBytes(long maxRepositoryBytes) { + this.maxRepositoryBytes = positive(maxRepositoryBytes, "maxRepositoryBytes"); + } + + /** + * 获取仓库文件对象数量上限。 + * + * @return 文件对象数量 + */ + public int getMaxRepositoryFiles() { + return maxRepositoryFiles; + } + + /** + * 设置仓库文件对象数量上限。 + * + * @param maxRepositoryFiles 文件对象数量 + */ + public void setMaxRepositoryFiles(int maxRepositoryFiles) { + this.maxRepositoryFiles = positive(maxRepositoryFiles, "maxRepositoryFiles"); + } + + /** + * 获取 Skill 候选数量上限。 + * + * @return 候选数量 + */ + public int getMaxCandidates() { + return maxCandidates; + } + + /** + * 设置 Skill 候选数量上限。 + * + * @param maxCandidates 候选数量 + */ + public void setMaxCandidates(int maxCandidates) { + this.maxCandidates = positive(maxCandidates, "maxCandidates"); + } + + /** + * 获取允许访问的 HTTPS 端口。 + * + * @return 端口集合 + */ + public Set getAllowedPorts() { + return allowedPorts; + } + + /** + * 设置允许访问的 HTTPS 端口。 + * + * @param allowedPorts 端口集合 + */ + public void setAllowedPorts(Set allowedPorts) { + LinkedHashSet normalized = new LinkedHashSet<>(); + if (allowedPorts != null) { + allowedPorts.stream().filter(port -> port != null && port > 0 && port <= 65_535) + .forEach(normalized::add); + } + this.allowedPorts = normalized.isEmpty() ? new LinkedHashSet<>(Set.of(443)) : normalized; + } + + /** + * 获取可信私网 Git 主机。 + * + * @return 小写主机集合 + */ + public Set getTrustedPrivateHosts() { + return trustedPrivateHosts; + } + + /** + * 设置可信私网 Git 主机。 + * + * @param trustedPrivateHosts 主机集合 + */ + public void setTrustedPrivateHosts(Set trustedPrivateHosts) { + LinkedHashSet normalized = new LinkedHashSet<>(); + if (trustedPrivateHosts != null) { + trustedPrivateHosts.stream().filter(host -> host != null && !host.isBlank()) + .map(host -> host.trim().toLowerCase(java.util.Locale.ROOT)) + .forEach(normalized::add); + } + this.trustedPrivateHosts = normalized; + } + + private static int positive(int value, String name) { + if (value <= 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return value; + } + + private static long positive(long value, String name) { + if (value <= 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return value; + } + + private static Duration positive(Duration value, String name) { + if (value == null || value.isZero() || value.isNegative()) { + throw new IllegalArgumentException(name + " must be positive"); + } + return value; + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitImportService.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitImportService.java new file mode 100644 index 00000000..e977a69f --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitImportService.java @@ -0,0 +1,168 @@ +package tech.easyflow.skill.gitimport; + +import org.eclipse.jgit.lib.Repository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.skill.imports.SkillImportPreview; +import tech.easyflow.skill.imports.SkillImportService; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * 编排 Git 仓库扫描、固定提交复核与标准 Skill 导入预览。 + */ +@Service +public class SkillGitImportService { + + private static final Logger LOG = LoggerFactory.getLogger(SkillGitImportService.class); + private static final int MAX_PREPARE_COUNT = SkillImportService.MAX_BATCH_SKILL_COUNT; + + private final GitRepositoryAccessPolicy accessPolicy; + private final SkillGitRepositoryReader repositoryReader; + private final SkillGitCandidateScanner candidateScanner; + private final SkillGitScanStore scanStore; + private final SkillGitScanRateLimiter scanRateLimiter; + private final SkillImportService skillImportService; + + /** + * 创建 Git Skill 导入编排服务。 + * + * @param accessPolicy 仓库地址访问策略 + * @param repositoryReader Git 仓库读取器 + * @param candidateScanner Skill 候选扫描器 + * @param scanStore 短期扫描会话仓库 + * @param scanRateLimiter 仓库扫描频控器 + * @param skillImportService 标准 Skill ZIP 导入服务 + */ + public SkillGitImportService(GitRepositoryAccessPolicy accessPolicy, + SkillGitRepositoryReader repositoryReader, + SkillGitCandidateScanner candidateScanner, + SkillGitScanStore scanStore, + SkillGitScanRateLimiter scanRateLimiter, + SkillImportService skillImportService) { + this.accessPolicy = accessPolicy; + this.repositoryReader = repositoryReader; + this.candidateScanner = candidateScanner; + this.scanStore = scanStore; + this.scanRateLimiter = scanRateLimiter; + this.skillImportService = skillImportService; + } + + /** + * 扫描任意受支持的 HTTPS Git 仓库并登记短期固定提交结果。 + * + * @param repositoryUrl Git 仓库地址，可省略 .git 后缀 + * @return 候选扫描结果 + */ + public SkillGitScanResult scan(String repositoryUrl) { + scanRateLimiter.check(); + String normalizedUrl = accessPolicy.normalizeRepositoryUrl(repositoryUrl); + try (SkillGitRepositoryReader.Snapshot snapshot = repositoryReader.openDefault(normalizedUrl)) { + SkillGitRepositoryScan repositoryScan = candidateScanner.scan(snapshot); + SkillGitScanResult.RepositoryInfo repository = new SkillGitScanResult.RepositoryInfo( + repositoryName(normalizedUrl), normalizedUrl, + Repository.shortenRefName(snapshot.branchRef()), snapshot.commitSha()); + return scanStore.create(repository, snapshot.branchRef(), repositoryScan.candidates()); + } + } + + /** + * 复核扫描时的固定提交，并把选中候选转换为既有标准包导入预览。 + * + * @param scanToken 扫描令牌 + * @param candidateIds 选中的候选 ID + * @return 与现有 ZIP 导入一致的预览列表 + */ + public List prepare(String scanToken, List candidateIds) { + validateCandidateIds(candidateIds); + return scanStore.withLockedSession(scanToken, + session -> prepareLocked(scanToken, session, List.copyOf(candidateIds))); + } + + private List prepareLocked(String scanToken, + SkillGitScanStore.Session session, + List candidateIds) { + SkillGitScanResult original = session.result(); + Set allowedIds = new HashSet<>(); + original.candidates().stream().filter(SkillGitScanResult.Candidate::importable) + .map(SkillGitScanResult.Candidate::candidateId).forEach(allowedIds::add); + if (!allowedIds.containsAll(candidateIds)) { + throw new BusinessException("所选 Git Skill 不存在或当前不可导入，请重新扫描"); + } + + List previews = new ArrayList<>(candidateIds.size()); + try (SkillGitRepositoryReader.Snapshot snapshot = repositoryReader.openCommit( + original.repository().url(), session.branchRef(), original.repository().commitSha())) { + SkillGitRepositoryScan current = candidateScanner.scan(snapshot); + Set currentImportableIds = new HashSet<>(); + current.candidates().stream().filter(SkillGitScanResult.Candidate::importable) + .map(SkillGitScanResult.Candidate::candidateId).forEach(currentImportableIds::add); + if (!currentImportableIds.containsAll(candidateIds)) { + throw new BusinessException("Git 仓库扫描结果已变化，请重新扫描"); + } + for (String candidateId : candidateIds) { + SkillGitScanResult.Candidate candidate = current.candidates().stream() + .filter(item -> item.candidateId().equals(candidateId)) + .findFirst().orElseThrow(() -> new BusinessException("Git Skill 候选不存在，请重新扫描")); + Path archive = candidateScanner.createStandardZip(snapshot, current, candidateId); + try { + previews.add(skillImportService.preview(candidate.name() + ".zip", archive.toFile())); + } finally { + candidateScanner.deleteArchive(archive); + } + } + scanStore.remove(scanToken); + return List.copyOf(previews); + } catch (RuntimeException exception) { + cancelPreparedPreviews(previews); + throw exception; + } + } + + private void cancelPreparedPreviews(List previews) { + for (SkillImportPreview preview : previews) { + if (preview == null || preview.getImportToken() == null || preview.getImportToken().isBlank()) { + continue; + } + try { + skillImportService.cancel(preview.getImportToken()); + } catch (RuntimeException cleanupFailure) { + LOG.error("回滚 Git Skill 导入预览失败，token={}", preview.getImportToken(), cleanupFailure); + } + } + } + + private void validateCandidateIds(List candidateIds) { + if (candidateIds == null || candidateIds.isEmpty()) { + throw new BusinessException("请选择要导入的 Git Skill"); + } + if (candidateIds.size() > MAX_PREPARE_COUNT) { + throw new BusinessException("单次最多导入 " + MAX_PREPARE_COUNT + " 个 Git Skill"); + } + if (candidateIds.stream().anyMatch(id -> id == null || !id.matches("^[a-fA-F0-9]{64}$"))) { + throw new BusinessException("Git Skill 候选参数格式不正确"); + } + if (new HashSet<>(candidateIds).size() != candidateIds.size()) { + throw new BusinessException("不能重复选择同一个 Git Skill"); + } + } + + private String repositoryName(String repositoryUrl) { + try { + String path = new URI(repositoryUrl).getPath(); + String name = path.substring(path.lastIndexOf('/') + 1); + return name.toLowerCase(java.util.Locale.ROOT).endsWith(".git") + ? name.substring(0, name.length() - 4) : name; + } catch (URISyntaxException | RuntimeException exception) { + throw new BusinessException("Git 仓库地址格式不正确"); + } + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitOperationExecutor.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitOperationExecutor.java new file mode 100644 index 00000000..cbb37bed --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitOperationExecutor.java @@ -0,0 +1,142 @@ +package tech.easyflow.skill.gitimport; + +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +/** + * 以有界线程池和总超时执行 Git 远程操作。 + */ +@Component +public class SkillGitOperationExecutor { + + private static final Logger LOG = LoggerFactory.getLogger(SkillGitOperationExecutor.class); + + private final SkillGitImportProperties properties; + private final ThreadPoolExecutor executor; + + /** + * 创建 Git 远程操作执行器。 + * + * @param properties Git 导入配置 + */ + public SkillGitOperationExecutor(SkillGitImportProperties properties) { + this.properties = properties; + int concurrency = properties.getMaxConcurrentOperations(); + this.executor = new ThreadPoolExecutor(concurrency, concurrency, + 30L, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory(), + new ThreadPoolExecutor.AbortPolicy()); + this.executor.allowCoreThreadTimeOut(true); + } + + /** + * 在并发与总超时边界内执行远程操作。 + * + * @param operation 远程操作 + * @param 返回值类型 + * @return 操作结果 + * @throws BusinessException 操作繁忙、超时或执行失败 + */ + public T execute(Callable operation) { + return execute(operation, ignored -> { + }); + } + + /** + * 在并发与总超时边界内执行远程操作，并清理超时后才产出的结果。 + * + * @param operation 远程操作 + * @param abandonedResultCleanup 超时或中断后结果清理器 + * @param 返回值类型 + * @return 操作结果 + * @throws BusinessException 操作繁忙、超时或执行失败 + */ + public T execute(Callable operation, Consumer abandonedResultCleanup) { + AtomicBoolean abandoned = new AtomicBoolean(); + AtomicReference produced = new AtomicReference<>(); + Future future; + try { + future = executor.submit(() -> { + T result = operation.call(); + produced.set(result); + cleanupIfAbandoned(abandoned, produced, abandonedResultCleanup); + return result; + }); + } catch (java.util.concurrent.RejectedExecutionException exception) { + throw new BusinessException(429, 42921, "Git 导入任务繁忙，请稍后重试", exception); + } + try { + T result = future.get(properties.getOperationTimeout().toMillis(), TimeUnit.MILLISECONDS); + produced.compareAndSet(result, null); + return result; + } catch (TimeoutException exception) { + abandoned.set(true); + future.cancel(true); + cleanupIfAbandoned(abandoned, produced, abandonedResultCleanup); + throw new BusinessException(504, 50421, "Git 仓库读取超时，请稍后重试", exception); + } catch (InterruptedException exception) { + abandoned.set(true); + future.cancel(true); + cleanupIfAbandoned(abandoned, produced, abandonedResultCleanup); + Thread.currentThread().interrupt(); + throw new BusinessException(503, 50321, "Git 仓库读取被中断，请稍后重试", exception); + } catch (ExecutionException exception) { + Throwable cause = exception.getCause(); + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } + if (cause instanceof Error error) { + throw error; + } + throw new BusinessException(502, 5021, "Git 仓库读取失败，请稍后重试", cause); + } + } + + private void cleanupIfAbandoned(AtomicBoolean abandoned, + AtomicReference produced, + Consumer abandonedResultCleanup) { + if (!abandoned.get()) { + return; + } + T result = produced.getAndSet(null); + if (result != null) { + try { + abandonedResultCleanup.accept(result); + } catch (RuntimeException exception) { + LOG.error("清理已放弃的 Git 远程操作结果失败", exception); + } + } + } + + /** + * 停止执行器并中断尚未结束的远程操作。 + */ + @PreDestroy + public void close() { + executor.shutdownNow(); + } + + private ThreadFactory threadFactory() { + AtomicInteger sequence = new AtomicInteger(); + return task -> { + Thread thread = new Thread(task, "easyflow-skill-git-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitRepositoryReader.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitRepositoryReader.java new file mode 100644 index 00000000..d3bfb03d --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitRepositoryReader.java @@ -0,0 +1,640 @@ +package tech.easyflow.skill.gitimport; + +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContexts; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.TransportConfigCallback; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.TagOpt; +import org.eclipse.jgit.transport.Transport; +import org.eclipse.jgit.transport.TransportHttp; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.transport.http.HttpConnectionFactory; +import org.eclipse.jgit.transport.http.HttpConnectionFactory2; +import org.eclipse.jgit.transport.http.apache.HttpClientConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.ProtocolException; +import java.net.Proxy; +import java.net.URL; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.KeyManager; +import javax.net.ssl.TrustManager; + +/** + * 基于 Eclipse JGit 的受控只读仓库读取器。 + */ +@Component +public class SkillGitRepositoryReader { + + private static final Logger LOG = LoggerFactory.getLogger(SkillGitRepositoryReader.class); + private static final String PINNED_REF = "refs/easyflow/skill-import"; + + private final GitRepositoryAccessPolicy accessPolicy; + private final SkillGitImportProperties properties; + private final SkillGitOperationExecutor operationExecutor; + + /** + * 创建 Git 仓库读取器。 + * + * @param accessPolicy 地址访问策略 + * @param properties Git 导入配置 + * @param operationExecutor Git 远程操作执行器 + */ + public SkillGitRepositoryReader(GitRepositoryAccessPolicy accessPolicy, + SkillGitImportProperties properties, + SkillGitOperationExecutor operationExecutor) { + this.accessPolicy = accessPolicy; + this.properties = properties; + this.operationExecutor = operationExecutor; + } + + /** + * 读取远程默认分支的当前固定提交。 + * + * @param repositoryUrl 规范化仓库地址 + * @return 临时只读仓库快照 + * @throws BusinessException 远程仓库不可读取或超过资源限制 + */ + public Snapshot openDefault(String repositoryUrl) { + return operationExecutor.execute(() -> openDefaultInternal(repositoryUrl), Snapshot::close); + } + + private Snapshot openDefaultInternal(String repositoryUrl) { + RemoteHead remoteHead = resolveRemoteHead(repositoryUrl); + return cloneBranch(repositoryUrl, remoteHead.branchRef(), remoteHead.commitSha()); + } + + /** + * 重新读取扫描时固定的提交；默认分支已变化时尝试按提交 SHA 获取。 + * + * @param repositoryUrl 规范化仓库地址 + * @param branchRef 扫描时默认分支完整引用 + * @param commitSha 扫描时完整提交 SHA + * @return 固定提交快照 + * @throws BusinessException 提交不可读取或仓库超过资源限制 + */ + public Snapshot openCommit(String repositoryUrl, String branchRef, String commitSha) { + return operationExecutor.execute( + () -> openCommitInternal(repositoryUrl, branchRef, commitSha), Snapshot::close); + } + + private Snapshot openCommitInternal(String repositoryUrl, String branchRef, String commitSha) { + Snapshot branchSnapshot = cloneBranch(repositoryUrl, branchRef, null); + if (commitSha.equals(branchSnapshot.commitSha())) { + return branchSnapshot; + } + try { + RefSpec pinned = new RefSpec().setForceUpdate(true) + .setSourceDestination(commitSha, PINNED_REF); + // Git.wrap 与快照共享 Repository；此处不能关闭包装器，否则会提前关闭后续扫描所需的仓库。 + Git.wrap(branchSnapshot.repository()).fetch() + .setRemote(Constants.DEFAULT_REMOTE_NAME) + .setRefSpecs(pinned) + .setDepth(1) + .setTagOpt(TagOpt.NO_TAGS) + .setTimeout(timeoutSeconds()) + .setTransportConfigCallback(transportCallback()) + .call(); + ObjectId pinnedId = branchSnapshot.repository().resolve(PINNED_REF); + if (pinnedId == null || !commitSha.equals(pinnedId.name())) { + throw new BusinessException("扫描时的 Git 提交已不可读取，请重新扫描"); + } + enforceRepositorySize(branchSnapshot.directory()); + return branchSnapshot.withCommit(commitSha); + } catch (GitAPIException | IOException exception) { + branchSnapshot.close(); + throw remoteFailure("扫描时的 Git 提交已不可读取，请重新扫描", exception); + } catch (RuntimeException exception) { + branchSnapshot.close(); + throw exception; + } + } + + private RemoteHead resolveRemoteHead(String repositoryUrl) { + try { + LsRemoteCommand command = Git.lsRemoteRepository() + .setRemote(repositoryUrl) + .setTimeout(timeoutSeconds()) + .setTransportConfigCallback(transportCallback()); + Map refs = command.callAsMap(); + Ref head = refs.get(Constants.HEAD); + if (head == null || head.getObjectId() == null) { + throw new BusinessException("Git 仓库没有可读取的默认分支"); + } + Ref branch = resolveHeadBranch(head, refs.values()); + if (branch == null) { + throw new BusinessException("无法识别 Git 仓库默认分支"); + } + return new RemoteHead(branch.getName(), head.getObjectId().name()); + } catch (GitAPIException exception) { + throw remoteFailure("无法读取 Git 仓库，请检查地址和访问权限", exception); + } + } + + private Ref resolveHeadBranch(Ref head, Collection refs) { + if (head.isSymbolic() && head.getTarget() != null + && head.getTarget().getName().startsWith(Constants.R_HEADS)) { + return head.getTarget(); + } + return refs.stream() + .filter(ref -> ref.getName().startsWith(Constants.R_HEADS)) + .filter(ref -> head.getObjectId().equals(ref.getObjectId())) + .sorted(Comparator.comparingInt(this::branchPriority).thenComparing(Ref::getName)) + .findFirst().orElse(null); + } + + private int branchPriority(Ref ref) { + return switch (ref.getName()) { + case Constants.R_HEADS + "main" -> 0; + case Constants.R_HEADS + Constants.MASTER -> 1; + default -> 2; + }; + } + + private Snapshot cloneBranch(String repositoryUrl, String branchRef, String expectedCommit) { + Path directory = createTemporaryDirectory(); + Path gitDirectory = directory.resolve("repository.git"); + Repository repository = null; + try { + Git git = Git.cloneRepository() + .setURI(repositoryUrl) + .setDirectory(gitDirectory.toFile()) + .setBare(true) + .setNoCheckout(true) + .setBranchesToClone(java.util.List.of(branchRef)) + .setBranch(branchRef) + .setDepth(1) + .setTagOption(TagOpt.NO_TAGS) + .setTimeout(timeoutSeconds()) + .setTransportConfigCallback(transportCallback()) + .call(); + repository = git.getRepository(); + ObjectId head = repository.resolve(Constants.HEAD); + if (head == null) { + repository.close(); + deleteDirectory(directory); + throw new BusinessException("Git 仓库默认分支没有可读取的提交"); + } + enforceRepositorySize(directory); + if (expectedCommit != null && !expectedCommit.equals(head.name())) { + repository.close(); + deleteDirectory(directory); + throw new BusinessException("Git 仓库默认分支在扫描期间发生变化，请重新扫描"); + } + return new Snapshot(repository, directory, branchRef, head.name()); + } catch (GitAPIException | IOException exception) { + if (repository != null) { + repository.close(); + } + deleteDirectory(directory); + throw remoteFailure("无法读取 Git 仓库，请检查地址和访问权限", exception); + } catch (RuntimeException exception) { + if (repository != null) { + repository.close(); + } + deleteDirectory(directory); + throw exception; + } + } + + private TransportConfigCallback transportCallback() { + HttpConnectionFactory factory = new PolicyHttpConnectionFactory( + accessPolicy, properties.getMaxRepositoryBytes(), timeoutMillis()); + int timeout = timeoutSeconds(); + return transport -> configureTransport(transport, factory, timeout); + } + + private void configureTransport(Transport transport, HttpConnectionFactory factory, int timeout) { + if (!(transport instanceof TransportHttp httpTransport)) { + throw new IllegalArgumentException("Git 仓库仅支持 HTTPS 传输"); + } + transport.setTimeout(timeout); + httpTransport.setHttpConnectionFactory(factory); + } + + private int timeoutSeconds() { + long seconds = Math.max(1L, properties.getTimeout().toSeconds()); + return (int) Math.min(Integer.MAX_VALUE, seconds); + } + + private int timeoutMillis() { + long millis = Math.max(1L, properties.getTimeout().toMillis()); + return (int) Math.min(Integer.MAX_VALUE, millis); + } + + private Path createTemporaryDirectory() { + try { + return Files.createTempDirectory("easyflow-skill-git-"); + } catch (IOException exception) { + throw new BusinessException(500, 500, "创建 Git 仓库临时目录失败", exception); + } + } + + private void enforceRepositorySize(Path directory) { + try (java.util.stream.Stream paths = Files.walk(directory)) { + long total = 0L; + java.util.Iterator iterator = paths.filter(Files::isRegularFile).iterator(); + while (iterator.hasNext()) { + total = Math.addExact(total, Files.size(iterator.next())); + if (total > properties.getMaxRepositoryBytes()) { + throw new BusinessException(413, 4132, "Git 仓库超过允许的临时存储大小"); + } + } + } catch (ArithmeticException exception) { + throw new BusinessException(413, 4132, "Git 仓库超过允许的临时存储大小"); + } catch (IOException exception) { + throw new BusinessException(500, 500, "检查 Git 仓库临时文件失败", exception); + } + } + + private BusinessException remoteFailure(String message, Exception exception) { + return new BusinessException(502, 5021, message, exception); + } + + private static void deleteDirectory(Path directory) { + if (directory == null || !Files.exists(directory)) { + return; + } + try (java.util.stream.Stream paths = Files.walk(directory)) { + paths.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException exception) { + LOG.warn("清理 Skill Git 临时文件失败，path={}", path, exception); + } + }); + } catch (IOException exception) { + LOG.warn("遍历 Skill Git 临时目录失败，path={}", directory, exception); + } + } + + /** + * 单次固定提交的临时仓库快照。 + */ + public static final class Snapshot implements AutoCloseable { + + private final Repository repository; + private final Path directory; + private final String branchRef; + private final String commitSha; + + Snapshot(Repository repository, Path directory, String branchRef, String commitSha) { + this.repository = repository; + this.directory = directory; + this.branchRef = branchRef; + this.commitSha = commitSha; + } + + /** + * 获取 JGit 仓库。 + * + * @return 只读仓库 + */ + public Repository repository() { + return repository; + } + + /** + * 获取临时目录。 + * + * @return 临时目录 + */ + public Path directory() { + return directory; + } + + /** + * 获取默认分支完整引用。 + * + * @return 分支引用 + */ + public String branchRef() { + return branchRef; + } + + /** + * 获取固定提交 SHA。 + * + * @return 完整提交 SHA + */ + public String commitSha() { + return commitSha; + } + + private Snapshot withCommit(String pinnedCommit) { + return new Snapshot(repository, directory, branchRef, pinnedCommit); + } + + /** + * 关闭仓库并清理临时目录。 + */ + @Override + public void close() { + repository.close(); + deleteDirectory(directory); + } + } + + private record RemoteHead(String branchRef, String commitSha) { + } + + private static final class PolicyHttpConnectionFactory implements HttpConnectionFactory2 { + + private final GitRepositoryAccessPolicy accessPolicy; + private final long maxResponseBytes; + private final AtomicLong responseBytes = new AtomicLong(); + private final Map pinnedAddresses = new ConcurrentHashMap<>(); + private final CloseableHttpClient httpClient; + + private PolicyHttpConnectionFactory(GitRepositoryAccessPolicy accessPolicy, + long maxResponseBytes, + int timeoutMillis) { + this.accessPolicy = accessPolicy; + this.maxResponseBytes = maxResponseBytes; + Registry registry = RegistryBuilder.create() + .register("https", new SSLConnectionSocketFactory( + SSLContexts.createSystemDefault(), + SSLConnectionSocketFactory.getDefaultHostnameVerifier())) + .register("http", PlainConnectionSocketFactory.INSTANCE) + .build(); + PoolingHttpClientConnectionManager connectionManager = + new PoolingHttpClientConnectionManager(registry, this::resolvePinnedAddress); + connectionManager.setMaxTotal(4); + connectionManager.setDefaultMaxPerRoute(2); + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(timeoutMillis) + .setConnectionRequestTimeout(timeoutMillis) + .setSocketTimeout(timeoutMillis) + .setRedirectsEnabled(false) + .build(); + this.httpClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .setDefaultRequestConfig(requestConfig) + .disableAutomaticRetries() + .disableRedirectHandling() + .build(); + } + + @Override + public HttpConnection create(URL url) throws IOException { + return create(url, Proxy.NO_PROXY); + } + + @Override + public HttpConnection create(URL url, Proxy proxy) throws IOException { + if (proxy != null && !Proxy.NO_PROXY.equals(proxy)) { + throw new IOException("Git repository proxy access is disabled"); + } + GitRepositoryAccessPolicy.ResolvedConnection resolved = accessPolicy.resolveConnection(url); + pinnedAddresses.putIfAbsent(resolved.host(), resolved.addresses()); + return limited(new HttpClientConnection(url.toString(), Proxy.NO_PROXY, httpClient)); + } + + /** + * 创建一次 JGit HTTP 会话并在结束时释放专用客户端。 + * + * @return Git HTTP 会话 + */ + @Override + public GitSession newSession() { + return new GitSession() { + @Override + public HttpConnection configure(HttpConnection connection, boolean sslVerify) throws IOException { + if (!sslVerify) { + throw new IOException("Git repository TLS verification must remain enabled"); + } + return connection; + } + + @Override + public void close() { + try { + httpClient.close(); + } catch (IOException exception) { + LOG.warn("关闭 Skill Git HTTP 客户端失败", exception); + } + } + }; + } + + private InetAddress[] resolvePinnedAddress(String host) throws UnknownHostException { + InetAddress[] addresses = pinnedAddresses.get(host.toLowerCase(Locale.ROOT)); + if (addresses == null || addresses.length == 0) { + throw new UnknownHostException("Git repository host was not approved: " + host); + } + return addresses.clone(); + } + + private HttpConnection limited(HttpConnection connection) { + return new LimitedHttpConnection(connection, responseBytes, maxResponseBytes); + } + } + + /** + * 对 JGit HTTP 响应流实施单次远程操作的累计字节上限。 + */ + private static final class LimitedHttpConnection implements HttpConnection { + + private final HttpConnection delegate; + private final AtomicLong totalBytes; + private final long maximumBytes; + private InputStream inputStream; + + private LimitedHttpConnection(HttpConnection delegate, AtomicLong totalBytes, long maximumBytes) { + this.delegate = delegate; + this.totalBytes = totalBytes; + this.maximumBytes = maximumBytes; + } + + @Override + public int getResponseCode() throws IOException { + return delegate.getResponseCode(); + } + + @Override + public URL getURL() { + return delegate.getURL(); + } + + @Override + public String getResponseMessage() throws IOException { + return delegate.getResponseMessage(); + } + + @Override + public Map> getHeaderFields() { + return delegate.getHeaderFields(); + } + + @Override + public void setRequestProperty(String key, String value) { + delegate.setRequestProperty(key, value); + } + + @Override + public void setRequestMethod(String method) throws ProtocolException { + delegate.setRequestMethod(method); + } + + @Override + public void setUseCaches(boolean useCaches) { + delegate.setUseCaches(useCaches); + } + + @Override + public void setConnectTimeout(int timeout) { + delegate.setConnectTimeout(timeout); + } + + @Override + public void setReadTimeout(int timeout) { + delegate.setReadTimeout(timeout); + } + + @Override + public String getContentType() { + return delegate.getContentType(); + } + + @Override + public InputStream getInputStream() throws IOException { + if (inputStream == null) { + int contentLength = delegate.getContentLength(); + if (contentLength > 0 && totalBytes.get() + contentLength > maximumBytes) { + throw new IOException("Git repository response exceeds the configured size limit"); + } + inputStream = new FilterInputStream(delegate.getInputStream()) { + @Override + public int read() throws IOException { + int value = super.read(); + if (value >= 0) { + addBytes(1L); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int count = super.read(buffer, offset, length); + if (count > 0) { + addBytes(count); + } + return count; + } + }; + } + return inputStream; + } + + private void addBytes(long count) throws IOException { + if (totalBytes.addAndGet(count) > maximumBytes) { + throw new IOException("Git repository response exceeds the configured size limit"); + } + } + + @Override + public String getHeaderField(String name) { + return delegate.getHeaderField(name); + } + + @Override + public List getHeaderFields(String name) { + return delegate.getHeaderFields(name); + } + + @Override + public int getContentLength() { + return delegate.getContentLength(); + } + + @Override + public void setInstanceFollowRedirects(boolean followRedirects) { + delegate.setInstanceFollowRedirects(followRedirects); + } + + @Override + public void setDoOutput(boolean doOutput) { + delegate.setDoOutput(doOutput); + } + + @Override + public void setFixedLengthStreamingMode(int contentLength) { + delegate.setFixedLengthStreamingMode(contentLength); + } + + @Override + public OutputStream getOutputStream() throws IOException { + return delegate.getOutputStream(); + } + + @Override + public void setChunkedStreamingMode(int chunkLength) { + delegate.setChunkedStreamingMode(chunkLength); + } + + @Override + public String getRequestMethod() { + return delegate.getRequestMethod(); + } + + @Override + public boolean usingProxy() { + return delegate.usingProxy(); + } + + @Override + public void connect() throws IOException { + delegate.connect(); + } + + @Override + public void configure(KeyManager[] keyManagers, TrustManager[] trustManagers, + SecureRandom random) + throws NoSuchAlgorithmException, KeyManagementException { + delegate.configure(keyManagers, trustManagers, random); + } + + @Override + public void setHostnameVerifier(HostnameVerifier verifier) + throws NoSuchAlgorithmException, KeyManagementException { + delegate.setHostnameVerifier(verifier); + } + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitRepositoryScan.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitRepositoryScan.java new file mode 100644 index 00000000..4310f120 --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitRepositoryScan.java @@ -0,0 +1,38 @@ +package tech.easyflow.skill.gitimport; + +import org.eclipse.jgit.lib.ObjectId; + +import java.util.List; +import java.util.Map; + +/** + * 单次仓库读取期间的候选扫描数据。 + * + * @param candidates 对外候选列表 + * @param filesByCandidate 候选对应的普通文件对象 + */ +record SkillGitRepositoryScan( + List candidates, + Map> filesByCandidate) { + + /** + * 创建不可变扫描数据。 + */ + SkillGitRepositoryScan { + candidates = List.copyOf(candidates); + filesByCandidate = Map.copyOf(filesByCandidate); + } + + /** + * 固定提交中的普通文件对象。 + * + * @param path 仓库相对路径 + * @param relativePath Skill 根目录相对路径 + * @param objectId Git Blob ID + * @param size 原始字节数 + * @param fileModeBits Git 文件模式 + */ + record RepositoryFile(String path, String relativePath, ObjectId objectId, + long size, int fileModeBits) { + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanRateLimiter.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanRateLimiter.java new file mode 100644 index 00000000..0b4280b4 --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanRateLimiter.java @@ -0,0 +1,80 @@ +package tech.easyflow.skill.gitimport; + +import com.alicp.jetcache.Cache; +import com.alicp.jetcache.CacheInvokeException; +import com.alicp.jetcache.CacheResult; +import com.alicp.jetcache.CacheResultCode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.util.concurrent.TimeUnit; + +/** + * 对同一租户账号的 Git 仓库扫描实施短周期频控。 + */ +@Component +public class SkillGitScanRateLimiter { + + private static final Logger LOG = LoggerFactory.getLogger(SkillGitScanRateLimiter.class); + private static final String CACHE_PREFIX = "skill:git-scan:rate:"; + + private final Cache defaultCache; + private final SkillGitImportProperties properties; + + /** + * 创建仓库扫描频控器。 + * + * @param defaultCache 平台默认缓存 + * @param properties Git 导入配置 + */ + public SkillGitScanRateLimiter(@Qualifier("defaultCache") Cache defaultCache, + SkillGitImportProperties properties) { + this.defaultCache = defaultCache; + this.properties = properties; + } + + /** + * 检查当前账号是否允许发起新的仓库扫描。 + * + * @throws BusinessException 未登录、请求过于频繁或缓存不可用 + */ + public void check() { + LoginAccount account = SaTokenUtil.getLoginAccount(); + if (account == null || account.getTenantId() == null || account.getId() == null) { + throw new BusinessException(401, 401, "未登录或登录态无效"); + } + check(account); + } + + /** + * 使用已认证账号执行频控检查，供契约测试复用。 + * + * @param account 已认证账号 + * @throws BusinessException 请求过于频繁或缓存不可用 + */ + void check(LoginAccount account) { + long intervalMillis = properties.getScanMinimumInterval().toMillis(); + String key = CACHE_PREFIX + account.getTenantId() + ":" + account.getId(); + try { + CacheResult result = defaultCache.PUT_IF_ABSENT( + key, Boolean.TRUE, intervalMillis, TimeUnit.MILLISECONDS); + if (result.getResultCode() == CacheResultCode.SUCCESS) { + return; + } + if (result.getResultCode() == CacheResultCode.EXISTS) { + throw new BusinessException(429, 42922, "Git 仓库扫描过于频繁，请稍后重试"); + } + LOG.error("Git 仓库扫描频控缓存写入失败，code={}, message={}", + result.getResultCode(), result.getMessage()); + throw new BusinessException(503, 50322, "Git 仓库扫描暂不可用，请稍后重试"); + } catch (CacheInvokeException exception) { + LOG.error("Git 仓库扫描频控缓存访问失败", exception); + throw new BusinessException(503, 50322, "Git 仓库扫描暂不可用，请稍后重试", exception); + } + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanResult.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanResult.java new file mode 100644 index 00000000..65cfa817 --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanResult.java @@ -0,0 +1,131 @@ +package tech.easyflow.skill.gitimport; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** + * Git 仓库 Skill 候选扫描结果。 + * + * @param scanToken 短期扫描令牌 + * @param repository 仓库固定提交信息 + * @param summary 候选数量摘要 + * @param candidates 候选列表 + * @param expiresAt 令牌过期时间 + */ +public record SkillGitScanResult( + String scanToken, + RepositoryInfo repository, + Summary summary, + List candidates, + Date expiresAt) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 创建不可变扫描结果。 + */ + public SkillGitScanResult { + candidates = candidates == null ? List.of() : List.copyOf(candidates); + expiresAt = expiresAt == null ? null : new Date(expiresAt.getTime()); + } + + /** + * 返回防御性复制的过期时间。 + * + * @return 过期时间 + */ + @Override + public Date expiresAt() { + return expiresAt == null ? null : new Date(expiresAt.getTime()); + } + + /** + * 固定提交的仓库信息。 + * + * @param name 仓库名称 + * @param url 规范化仓库地址 + * @param defaultBranch 默认分支 + * @param commitSha 完整提交 SHA + */ + public record RepositoryInfo(String name, String url, String defaultBranch, + String commitSha) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + } + + /** + * 候选数量摘要。 + * + * @param discovered 发现数量 + * @param importable 可导入数量 + * @param needsAttention 需处理数量 + */ + public record Summary(int discovered, int importable, int needsAttention) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + } + + /** + * 单个 Skill 候选。 + * + * @param candidateId 固定提交内稳定候选 ID + * @param path 仓库相对路径 + * @param name 标准 Skill 名称 + * @param description 用途描述 + * @param status 候选状态 + * @param resourceCount 资源文件数量 + * @param totalBytes 候选总字节数 + * @param issues 校验问题 + */ + public record Candidate(String candidateId, String path, String name, String description, + CandidateStatus status, int resourceCount, long totalBytes, + List issues) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * 创建不可变候选。 + */ + public Candidate { + issues = issues == null ? List.of() : List.copyOf(issues); + } + + /** + * 判断候选是否可以进入标准包预检。 + * + * @return 可导入时为 true + */ + public boolean importable() { + return status == CandidateStatus.IMPORTABLE; + } + } + + /** + * 候选状态。 + */ + public enum CandidateStatus { + /** 可进入标准包预检。 */ + IMPORTABLE, + /** 需修正仓库内容后重新扫描。 */ + NEEDS_ATTENTION + } + + /** + * 候选校验问题。 + * + * @param code 稳定问题码 + * @param message 用户可读说明 + * @param path 问题文件路径 + */ + public record Issue(String code, String message, String path) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanStore.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanStore.java new file mode 100644 index 00000000..41076cbc --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/gitimport/SkillGitScanStore.java @@ -0,0 +1,150 @@ +package tech.easyflow.skill.gitimport; + +import com.alicp.jetcache.AutoReleaseLock; +import com.alicp.jetcache.Cache; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +/** + * 保存短期 Git 扫描会话并约束租户、账号与并发使用。 + */ +@Service +public class SkillGitScanStore { + + private static final String CACHE_PREFIX = "skill:git-scan:"; + + private final Cache defaultCache; + private final SkillGitImportProperties properties; + + /** + * 创建 Git 扫描会话仓库。 + * + * @param defaultCache 平台默认缓存 + * @param properties Git 导入配置 + */ + public SkillGitScanStore(@Qualifier("defaultCache") Cache defaultCache, + SkillGitImportProperties properties) { + this.defaultCache = defaultCache; + this.properties = properties; + } + + /** + * 创建绑定当前账号的扫描会话。 + * + * @param repository 仓库固定提交信息 + * @param branchRef 默认分支完整引用 + * @param candidates 候选列表 + * @return 可返回给前端的扫描结果 + */ + public SkillGitScanResult create(SkillGitScanResult.RepositoryInfo repository, + String branchRef, + List candidates) { + LoginAccount account = requireAccount(); + String token = UUID.randomUUID().toString().replace("-", ""); + Date expiresAt = new Date(System.currentTimeMillis() + properties.getScanTtl().toMillis()); + int importable = (int) candidates.stream().filter(SkillGitScanResult.Candidate::importable).count(); + SkillGitScanResult result = new SkillGitScanResult(token, repository, + new SkillGitScanResult.Summary(candidates.size(), importable, candidates.size() - importable), + candidates, expiresAt); + Session session = new Session(account.getTenantId(), account.getId(), branchRef, result); + long ttlSeconds = Math.max(1L, properties.getScanTtl().toSeconds()); + defaultCache.put(cacheKey(token), session, ttlSeconds, TimeUnit.SECONDS); + return result; + } + + /** + * 在扫描会话单次锁内执行业务动作。 + * + * @param token 扫描令牌 + * @param action 受锁保护的动作 + * @param 返回值类型 + * @return 动作结果 + */ + public T withLockedSession(String token, Function action) { + validateToken(token); + LoginAccount account = requireAccount(); + try (AutoReleaseLock lock = defaultCache.tryLock(lockKey(token), 5, TimeUnit.MINUTES)) { + if (lock == null) { + throw new BusinessException("Git 仓库导入正在处理中，请勿重复提交"); + } + return action.apply(findOwned(token, account)); + } + } + + /** + * 删除已成功转换为标准导入预览的扫描会话。 + * + * @param token 扫描令牌 + */ + public void remove(String token) { + validateToken(token); + defaultCache.remove(cacheKey(token)); + } + + private Session findOwned(String token, LoginAccount account) { + Object cached = defaultCache.get(cacheKey(token)); + if (!(cached instanceof Session session)) { + throw new BusinessException(404, 404, "Git 仓库扫描结果不存在或已过期，请重新扫描"); + } + if (!account.getTenantId().equals(session.tenantId()) + || !account.getId().equals(session.accountId())) { + throw new BusinessException(403, 403, "无权限使用该 Git 仓库扫描结果"); + } + Date expiresAt = session.result().expiresAt(); + if (expiresAt == null || !expiresAt.after(new Date())) { + defaultCache.remove(cacheKey(token)); + throw new BusinessException("Git 仓库扫描结果已过期，请重新扫描"); + } + return session; + } + + private LoginAccount requireAccount() { + LoginAccount account = SaTokenUtil.getLoginAccount(); + if (account == null || account.getId() == null || account.getTenantId() == null) { + throw new BusinessException(401, 401, "未登录或登录态无效"); + } + return account; + } + + private void validateToken(String token) { + if (token == null || !token.matches("^[a-fA-F0-9]{32}$")) { + throw new BusinessException("Git 仓库扫描令牌格式不正确"); + } + } + + private String cacheKey(String token) { + return CACHE_PREFIX + token; + } + + private String lockKey(String token) { + return CACHE_PREFIX + "lock:" + token; + } + + /** + * 可序列化的短期扫描会话。 + * + * @param tenantId 租户 ID + * @param accountId 账号 ID + * @param branchRef 默认分支完整引用 + * @param result 对外扫描结果 + */ + public record Session(java.math.BigInteger tenantId, + java.math.BigInteger accountId, + String branchRef, + SkillGitScanResult result) implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportPreview.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportPreview.java index 22d649a6..b7134a4e 100644 --- a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportPreview.java +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportPreview.java @@ -11,6 +11,7 @@ import tech.easyflow.skill.validation.SkillValidationIssue; public class SkillImportPreview { private List skills = new ArrayList<>(); + private String sourceName; private String importToken; private Date expiresAt; private List issues = new ArrayList<>(); @@ -33,6 +34,24 @@ public class SkillImportPreview { this.skills = skills == null ? new ArrayList<>() : skills; } + /** + * 获取本预览对应的原始导入包名称。 + * + * @return 原始导入包名称 + */ + public String getSourceName() { + return sourceName; + } + + /** + * 设置本预览对应的原始导入包名称。 + * + * @param sourceName 原始导入包名称 + */ + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } + public String getImportToken() { return importToken; } public void setImportToken(String importToken) { this.importToken = importToken; } public Date getExpiresAt() { return expiresAt; } diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportService.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportService.java index 52327f01..079655a7 100644 --- a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportService.java +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportService.java @@ -2,6 +2,7 @@ package tech.easyflow.skill.imports; import tech.easyflow.skill.entity.Skill; +import java.io.File; import java.util.List; import org.springframework.web.multipart.MultipartFile; @@ -10,6 +11,17 @@ import org.springframework.web.multipart.MultipartFile; */ public interface SkillImportService { + /** 单次批量预检或确认允许处理的最大 Skill 数量。 */ + int MAX_BATCH_SKILL_COUNT = 50; + + /** + * 批量预检上传的标准 Skill ZIP，并将多 Skill 包拆分为独立预览。 + * + * @param files 标准 Skill ZIP 列表 + * @return 每个 Skill 对应一个 importToken 的预览列表 + */ + List previewBatch(List files); + /** * 上传并创建可单次确认的导入预览。 * @@ -18,6 +30,15 @@ public interface SkillImportService { */ SkillImportPreview preview(MultipartFile file); + /** + * 暂存并预检后端生成的标准 Skill ZIP。 + * + * @param originalName 用于结果展示的原始文件名 + * @param file 后端受控临时 ZIP + * @return 导入预览与 importToken + */ + SkillImportPreview preview(String originalName, File file); + /** * 使用单次 importToken 确认导入。 * diff --git a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportServiceImpl.java b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportServiceImpl.java index cbbc1675..d6e0883c 100644 --- a/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportServiceImpl.java +++ b/easyflow-modules/easyflow-module-skill/src/main/java/tech/easyflow/skill/imports/SkillImportServiceImpl.java @@ -2,9 +2,12 @@ package tech.easyflow.skill.imports; import com.easyagents.skill.codec.SkillPackageReadOptions; import com.easyagents.skill.codec.SkillPackageReadResult; +import com.easyagents.skill.codec.SkillPackageWriteOptions; import com.easyagents.skill.codec.ZipSkillPackageCodec; import com.easyagents.skill.exception.SkillPackageException; import com.easyagents.skill.model.SkillDocument; +import com.easyagents.skill.model.SkillPackage; +import com.easyagents.skill.model.SkillPackageLayout; import com.easyagents.skill.model.SkillPackageLimits; import com.easyagents.skill.model.SkillResourceKind; import com.easyagents.skill.store.SkillContentStage; @@ -38,10 +41,15 @@ import tech.easyflow.system.enums.ResourceAction; import tech.easyflow.system.enums.VisibilityScope; import tech.easyflow.system.service.ResourceAccessService; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.math.BigInteger; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -56,7 +64,7 @@ import java.util.Set; public class SkillImportServiceImpl implements SkillImportService { private static final Logger LOG = LoggerFactory.getLogger(SkillImportServiceImpl.class); - private static final int MAX_RENAMES = 20; + private static final int MAX_RENAMES = MAX_BATCH_SKILL_COUNT; private static final int NAME_CONFLICT_ERROR_CODE = 4092; private static final String NAME_UNAVAILABLE_REASON = "NAME_UNAVAILABLE"; @@ -99,6 +107,117 @@ public class SkillImportServiceImpl implements SkillImportService { return hasExactlyOneSkill(decoded) ? buildPreview(decoded, null) : invalidSkillCountPreview(); } + /** + * {@inheritDoc} + */ + @Override + public List previewBatch(List files) { + if (files == null || files.isEmpty()) { + throw new BusinessException("请选择至少一个 Skill ZIP 文件"); + } + if (files.size() > MAX_BATCH_SKILL_COUNT) { + throw batchLimitExceeded(); + } + + List prepared = new ArrayList<>(); + int totalSkills = 0; + try { + for (MultipartFile file : files) { + PreparedUpload upload = previewUploadedPackage( + file, MAX_BATCH_SKILL_COUNT - totalSkills); + prepared.addAll(upload.previews()); + totalSkills += upload.skillCount(); + } + return List.copyOf(prepared); + } catch (RuntimeException exception) { + cancelPreparedPreviews(prepared); + throw exception; + } + } + + /** + * 预检一个上传包，并在需要时拆分为独立 Skill 预览。 + * + * @param file 上传的标准 ZIP + * @param remainingSkills 当前批次剩余可接收 Skill 数 + * @return 本包产生的独立预览及 Skill 数量 + */ + private PreparedUpload previewUploadedPackage(MultipartFile file, int remainingSkills) { + validateUpload(file); + String sourceName = file.getOriginalFilename(); + SkillPackageReadResult reportOnly; + try (InputStream input = file.getInputStream()) { + reportOnly = new ZipSkillPackageCodec(new PreviewContentStore()) + .decode(input, SkillPackageReadOptions.reportOnly()); + } catch (SkillPackageException exception) { + return new PreparedUpload(List.of(failedPreview(exception, sourceName)), 0); + } catch (IOException exception) { + LOG.error("读取标准 Skill ZIP 失败，file={}", sourceName, exception); + throw new BusinessException(500, 500, "读取 Skill 导入包失败", exception); + } + + int skillCount = reportOnly.getSkillPackage().getSkills().size(); + if (skillCount > remainingSkills) { + throw batchLimitExceeded(); + } + if (skillCount == 1) { + SkillImportPreview preview = preview(file); + preview.setSourceName(sourceName); + return new PreparedUpload(List.of(preview), 1); + } + if (reportOnly.getValidationReport().hasErrors()) { + return new PreparedUpload(List.of(failedPreview(reportOnly, sourceName)), skillCount); + } + return splitAndPreview(file, sourceName, skillCount); + } + + /** + * 将已通过整包校验的多 Skill ZIP 拆成独立标准包并逐一暂存。 + * + * @param file 原始上传 ZIP + * @param sourceName 原始文件名 + * @param skillCount 已解析的 Skill 数量 + * @return 每个 Skill 对应一个 token 的预览结果 + */ + private PreparedUpload splitAndPreview(MultipartFile file, String sourceName, int skillCount) { + List prepared = new ArrayList<>(skillCount); + try (ZipSkillPackageCodec codec = new ZipSkillPackageCodec(); + InputStream input = file.getInputStream()) { + SkillPackageReadResult decoded = codec.decode(input, SkillPackageReadOptions.defaults()); + for (com.easyagents.skill.model.Skill skill : decoded.getSkillPackage().getSkills()) { + Path archive = null; + try { + archive = Files.createTempFile("easyflow-skill-import-", ".zip"); + try (OutputStream output = Files.newOutputStream(archive, + StandardOpenOption.TRUNCATE_EXISTING)) { + codec.encode(new SkillPackage(SkillPackageLayout.SINGLE_DIRECTORY, List.of(skill)), + output, SkillPackageWriteOptions.defaults()); + } + SkillImportPreview preview = preview(sourceName, archive.toFile()); + if (!hasUsableSingleSkillPreview(preview)) { + cancelPreview(preview); + throw new BusinessException(500, 500, "拆分 Skill 导入包后预检失败，请重新打包后重试"); + } + preview.setSourceName(sourceName); + prepared.add(preview); + } finally { + deleteLocalArchive(archive); + } + } + return new PreparedUpload(List.copyOf(prepared), skillCount); + } catch (SkillPackageException exception) { + cancelPreparedPreviews(prepared); + return new PreparedUpload(List.of(failedPreview(exception, sourceName)), skillCount); + } catch (IOException exception) { + cancelPreparedPreviews(prepared); + LOG.error("拆分多 Skill ZIP 失败，file={}", sourceName, exception); + throw new BusinessException(500, 500, "拆分 Skill 导入包失败", exception); + } catch (RuntimeException exception) { + cancelPreparedPreviews(prepared); + throw exception; + } + } + /** * {@inheritDoc} */ @@ -109,24 +228,10 @@ public class SkillImportServiceImpl implements SkillImportService { String storedPath = null; try { storedPath = fileStorageService.save(file, "skill-imports/" + account.getTenantId()); - if (storedPath == null || storedPath.isBlank()) { - throw new BusinessException(500, 500, "Skill 导入临时包存储失败，请稍后重试"); - } - SkillPackageReadResult decoded; - try (InputStream input = fileStorageService.readStream(storedPath)) { - decoded = new ZipSkillPackageCodec(new PreviewContentStore()) - .decode(input, SkillPackageReadOptions.reportOnly()); - } - if (!hasExactlyOneSkill(decoded)) { - cleanupUnregisteredPath(storedPath); - return invalidSkillCountPreview(); - } - SkillImportStage stage = stageStore.create(storedPath, file.getOriginalFilename()); - SkillImportPreview preview = buildPreview(decoded, stage); - return preview; + return previewStoredPath(storedPath, file.getOriginalFilename()); } catch (SkillPackageException exception) { cleanupUnregisteredPath(storedPath); - return failedPreview(exception); + return failedPreview(exception, file.getOriginalFilename()); } catch (BusinessException exception) { cleanupUnregisteredPath(storedPath); throw exception; @@ -140,6 +245,50 @@ public class SkillImportServiceImpl implements SkillImportService { } } + /** + * {@inheritDoc} + */ + @Override + public SkillImportPreview preview(String originalName, File file) { + validateGeneratedFile(originalName, file); + LoginAccount account = requireAccount(); + String storedPath = null; + try { + storedPath = fileStorageService.save(file, "skill-imports/" + account.getTenantId()); + return previewStoredPath(storedPath, originalName); + } catch (SkillPackageException exception) { + cleanupUnregisteredPath(storedPath); + return failedPreview(exception, originalName); + } catch (BusinessException exception) { + cleanupUnregisteredPath(storedPath); + throw exception; + } catch (IOException exception) { + cleanupUnregisteredPath(storedPath); + LOG.error("读取后端生成的标准 Skill ZIP 失败，path={}", storedPath, exception); + throw new BusinessException(500, 500, "读取 Skill 导入包失败", exception); + } catch (RuntimeException exception) { + cleanupUnregisteredPath(storedPath); + throw exception; + } + } + + private SkillImportPreview previewStoredPath(String storedPath, String originalName) throws IOException { + if (storedPath == null || storedPath.isBlank()) { + throw new BusinessException(500, 500, "Skill 导入临时包存储失败，请稍后重试"); + } + SkillPackageReadResult decoded; + try (InputStream input = fileStorageService.readStream(storedPath)) { + decoded = new ZipSkillPackageCodec(new PreviewContentStore()) + .decode(input, SkillPackageReadOptions.reportOnly()); + } + if (!hasExactlyOneSkill(decoded)) { + cleanupUnregisteredPath(storedPath); + return invalidSkillCountPreview(originalName); + } + SkillImportStage stage = stageStore.create(storedPath, originalName); + return buildPreview(decoded, stage); + } + /** * {@inheritDoc} */ @@ -236,6 +385,7 @@ public class SkillImportServiceImpl implements SkillImportService { private SkillImportPreview buildPreview(SkillPackageReadResult decoded, SkillImportStage stage) { SkillImportPreview preview = new SkillImportPreview(); if (stage != null) { + preview.setSourceName(stage.getOriginalName()); preview.setImportToken(stage.getImportToken()); preview.setExpiresAt(stage.getExpiresAt()); } @@ -376,6 +526,19 @@ public class SkillImportServiceImpl implements SkillImportService { } } + private void validateGeneratedFile(String originalName, File file) { + if (file == null || !file.isFile() || file.length() == 0L) { + throw new BusinessException("Skill 导入文件不能为空"); + } + long limit = SkillPackageLimits.defaults().getMaxCompressedPackageBytes(); + if (file.length() > limit) { + throw new BusinessException(413, 4131, "Skill 导入文件超过 " + limit + " 字节限制"); + } + if (originalName == null || !originalName.toLowerCase(java.util.Locale.ROOT).endsWith(".zip")) { + throw new BusinessException("Skill 导入仅支持标准 .zip 文件"); + } + } + private void validateConfirmRequest(SkillImportConfirmRequest request) { if (request == null || request.getImportToken() == null || request.getImportToken().isBlank()) { throw new BusinessException("Skill 导入确认参数不能为空"); @@ -399,14 +562,33 @@ public class SkillImportServiceImpl implements SkillImportService { } private SkillImportPreview invalidSkillCountPreview() { + return invalidSkillCountPreview(null); + } + + /** + * 构建单 token 路径收到非单 Skill 包时的失败预览。 + * + * @param sourceName 原始文件名 + * @return 失败预览 + */ + private SkillImportPreview invalidSkillCountPreview(String sourceName) { SkillImportPreview preview = new SkillImportPreview(); + preview.setSourceName(sourceName); preview.setIssues(List.of(SkillValidationIssue.of( "ERROR", "STANDARD_PACKAGE_SKILL_COUNT", "每个标准 Skill ZIP 必须且只能包含一个 Skill", null))); return preview; } - private SkillImportPreview failedPreview(SkillPackageException exception) { + /** + * 将包解析异常转换为可展示的失败预览。 + * + * @param exception 包解析异常 + * @param sourceName 原始文件名 + * @return 失败预览 + */ + private SkillImportPreview failedPreview(SkillPackageException exception, String sourceName) { SkillImportPreview preview = new SkillImportPreview(); + preview.setSourceName(sourceName); preview.setSkills(List.of()); if (exception.getReport() == null) { preview.setIssues(List.of(SkillValidationIssue.of( @@ -424,6 +606,92 @@ public class SkillImportServiceImpl implements SkillImportService { return preview; } + /** + * 将报告模式的整包校验错误转换为失败预览。 + * + * @param decoded 报告模式解码结果 + * @param sourceName 原始文件名 + * @return 失败预览 + */ + private SkillImportPreview failedPreview(SkillPackageReadResult decoded, String sourceName) { + SkillImportPreview preview = new SkillImportPreview(); + preview.setSourceName(sourceName); + preview.setSkills(List.of()); + preview.setIssues(decoded.getValidationReport().getIssues().stream().map(source -> { + SkillValidationIssue issue = SkillValidationIssue.of(source.getSeverity().name(), source.getCode(), + source.getMessage(), source.getPath()); + issue.setLine(source.getLine()); + issue.setColumn(source.getColumn()); + issue.setSuggestion(source.getSuggestion()); + return issue; + }).toList()); + return preview; + } + + /** + * 判断拆分后的预览是否满足单 Skill token 不变量。 + * + * @param preview 拆分包预览 + * @return token、Skill 与校验结果均可用时返回 true + */ + private boolean hasUsableSingleSkillPreview(SkillImportPreview preview) { + return preview != null + && preview.getImportToken() != null + && !preview.getImportToken().isBlank() + && preview.getSkills().size() == 1 + && preview.getIssues().stream().noneMatch(issue -> "ERROR".equals(issue.getSeverity())); + } + + /** + * 创建统一的批量 Skill 数量超限异常。 + * + * @return 业务异常 + */ + private BusinessException batchLimitExceeded() { + return new BusinessException("单次最多导入 " + MAX_BATCH_SKILL_COUNT + " 个 Skill"); + } + + /** + * 尽力回滚一组已创建的预览 token。 + * + * @param previews 已创建的预览 + */ + private void cancelPreparedPreviews(List previews) { + previews.forEach(this::cancelPreview); + } + + /** + * 尽力回滚一个已创建的预览 token。 + * + * @param preview 已创建的预览 + */ + private void cancelPreview(SkillImportPreview preview) { + if (preview == null || preview.getImportToken() == null || preview.getImportToken().isBlank()) { + return; + } + try { + cancel(preview.getImportToken()); + } catch (RuntimeException cleanupException) { + LOG.error("回滚 Skill 导入预览失败，token={}", preview.getImportToken(), cleanupException); + } + } + + /** + * 尽力删除拆包过程中创建的本地临时 ZIP。 + * + * @param archive 本地临时 ZIP 路径 + */ + private void deleteLocalArchive(Path archive) { + if (archive == null) { + return; + } + try { + Files.deleteIfExists(archive); + } catch (IOException cleanupException) { + LOG.warn("清理本地 Skill 拆分临时包失败，path={}", archive, cleanupException); + } + } + private void cleanupUnregisteredPath(String path) { if (path == null || path.isBlank()) { return; @@ -500,4 +768,13 @@ public class SkillImportServiceImpl implements SkillImportService { @Override public InputStream open(String contentRef) { throw new UnsupportedOperationException(); } @Override public boolean exists(String contentRef) { return true; } } + + /** + * 单个上传包完成预检后的独立预览及实际 Skill 数量。 + * + * @param previews 独立预览列表 + * @param skillCount 实际解析出的 Skill 数量 + */ + private record PreparedUpload(List previews, int skillCount) { + } } diff --git a/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/GitRepositoryAccessPolicyTest.java b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/GitRepositoryAccessPolicyTest.java new file mode 100644 index 00000000..7d1ec262 --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/GitRepositoryAccessPolicyTest.java @@ -0,0 +1,103 @@ +package tech.easyflow.skill.gitimport; + +import org.junit.Before; +import org.junit.Test; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +/** + * {@link GitRepositoryAccessPolicy} 地址安全契约测试。 + */ +public class GitRepositoryAccessPolicyTest { + + private GitRepositoryAccessPolicy policy; + + /** + * 初始化不依赖公网 DNS 的可信测试主机。 + */ + @Before + public void setUp() { + SkillGitImportProperties properties = new SkillGitImportProperties(); + properties.setTrustedPrivateHosts(Set.of("git.example.test")); + policy = new GitRepositoryAccessPolicy(properties); + } + + /** + * 仓库地址允许省略 .git 后缀并移除结尾斜线。 + */ + @Test + public void acceptsHttpsRepositoryWithOptionalGitSuffix() { + assertEquals(policy.normalizeRepositoryUrl( + " https://git.example.test/team/skills/ "), + "https://git.example.test/team/skills"); + assertEquals(policy.normalizeRepositoryUrl( + "https://git.example.test/team/skills.git"), + "https://git.example.test/team/skills.git"); + assertEquals(policy.normalizeRepositoryUrl("https://192.0.1.1/team/skills.git"), + "https://192.0.1.1/team/skills.git"); + } + + /** + * 拒绝非 HTTPS、内嵌凭证和 URL 参数。 + */ + @Test + public void rejectsUnsafeRepositoryInputs() { + assertRejected("http://git.example.test/team/skills.git"); + assertRejected("https://user:secret@git.example.test/team/skills.git"); + assertRejected("https://git.example.test/team/skills.git?token=secret"); + assertRejected("https://git.example.test:8443/team/skills.git"); + assertRejected("https://127.0.0.1/team/skills.git"); + } + + /** + * 实际连接使用策略校验后返回的固定 DNS 地址。 + * + * @throws Exception 构造测试地址失败 + */ + @Test + public void resolvesAddressesForPinnedConnection() throws Exception { + SkillGitImportProperties properties = new SkillGitImportProperties(); + InetAddress publicAddress = InetAddress.getByAddress(new byte[]{93, (byte) 184, (byte) 216, 34}); + GitRepositoryAccessPolicy pinnedPolicy = new GitRepositoryAccessPolicy( + properties, host -> new InetAddress[]{publicAddress}); + + GitRepositoryAccessPolicy.ResolvedConnection resolved = pinnedPolicy.resolveConnection( + new URL("https://git.example.test/team/skills.git/info/refs")); + + assertEquals("git.example.test", resolved.host()); + assertEquals(publicAddress, resolved.addresses()[0]); + } + + /** + * 输入校验后主机重新绑定到内网时，实际连接阶段必须拒绝。 + * + * @throws Exception 构造测试地址失败 + */ + @Test + public void rejectsDnsRebindingBeforeConnection() throws Exception { + SkillGitImportProperties properties = new SkillGitImportProperties(); + InetAddress publicAddress = InetAddress.getByAddress(new byte[]{93, (byte) 184, (byte) 216, 34}); + InetAddress loopback = InetAddress.getByAddress(new byte[]{127, 0, 0, 1}); + AtomicInteger resolutions = new AtomicInteger(); + GitRepositoryAccessPolicy rebindingPolicy = new GitRepositoryAccessPolicy(properties, + host -> resolutions.getAndIncrement() == 0 + ? new InetAddress[]{publicAddress} : new InetAddress[]{loopback}); + + assertEquals("https://git.example.test/team/skills.git", + rebindingPolicy.normalizeRepositoryUrl("https://git.example.test/team/skills.git")); + assertThrows(IOException.class, () -> rebindingPolicy.resolveConnection( + new URL("https://git.example.test/team/skills.git/info/refs"))); + } + + private void assertRejected(String value) { + assertThrows(BusinessException.class, () -> policy.normalizeRepositoryUrl(value)); + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitCandidateScannerTest.java b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitCandidateScannerTest.java new file mode 100644 index 00000000..f8b299ff --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitCandidateScannerTest.java @@ -0,0 +1,104 @@ +package tech.easyflow.skill.gitimport; + +import org.eclipse.jgit.api.Git; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipInputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * {@link SkillGitCandidateScanner} 固定提交扫描与标准包组装测试。 + */ +public class SkillGitCandidateScannerTest { + + /** + * 发现根目录和嵌套 Skill，并把不可解析的 LFS 资源标为需处理。 + * + * @throws Exception 仓库或 ZIP 读写失败 + */ + @Test + public void discoversCandidatesAndCreatesIndependentStandardZip() throws Exception { + SkillGitImportProperties properties = new SkillGitImportProperties(); + SkillGitCandidateScanner scanner = new SkillGitCandidateScanner(properties); + Path directory = Files.createTempDirectory("skill-git-scanner-test-"); + try (Git git = Git.init().setDirectory(directory.toFile()).call()) { + write(directory.resolve("SKILL.md"), skill("root-skill", "根目录技能")); + write(directory.resolve("references/guide.md"), "# Guide\n"); + write(directory.resolve("nested-skill/SKILL.md"), skill("nested-skill", "嵌套技能")); + write(directory.resolve("nested-skill/assets/model.bin"), + "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 10\n"); + git.add().addFilepattern(".").call(); + String commitSha = git.commit().setMessage("test") + .setAuthor("EasyFlow", "test@easyflow.local") + .setCommitter("EasyFlow", "test@easyflow.local").call().getName(); + SkillGitRepositoryReader.Snapshot snapshot = new SkillGitRepositoryReader.Snapshot( + git.getRepository(), directory, "refs/heads/master", commitSha); + + SkillGitRepositoryScan scan = scanner.scan(snapshot); + + assertEquals(scan.candidates().size(), 2); + SkillGitScanResult.Candidate root = candidate(scan, "root-skill"); + SkillGitScanResult.Candidate nested = candidate(scan, "nested-skill"); + assertTrue(root.importable()); + assertFalse(nested.importable()); + assertTrue(nested.issues().stream() + .anyMatch(issue -> "GIT_LFS_UNRESOLVED".equals(issue.code()))); + + Path archive = scanner.createStandardZip(snapshot, scan, root.candidateId()); + try { + assertEquals(zipEntries(archive), + Set.of("root-skill/SKILL.md", "root-skill/references/guide.md")); + } finally { + scanner.deleteArchive(archive); + } + // Snapshot 与 Git 共享 Repository，交给 Git 的 try-with-resources 统一关闭。 + } finally { + deleteDirectory(directory); + } + } + + private SkillGitScanResult.Candidate candidate(SkillGitRepositoryScan scan, String name) { + return scan.candidates().stream().filter(item -> name.equals(item.name())).findFirst() + .orElseThrow(); + } + + private Set zipEntries(Path archive) throws IOException { + Set entries = new HashSet<>(); + try (ZipInputStream input = new ZipInputStream(Files.newInputStream(archive))) { + java.util.zip.ZipEntry entry; + while ((entry = input.getNextEntry()) != null) { + entries.add(entry.getName()); + } + } + return entries; + } + + private String skill(String name, String description) { + return "---\nname: " + name + "\ndescription: " + description + "\n---\n# " + name + "\n"; + } + + private void write(Path path, String content) throws IOException { + Files.createDirectories(path.getParent()); + Files.writeString(path, content, StandardCharsets.UTF_8); + } + + private void deleteDirectory(Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (java.util.stream.Stream paths = Files.walk(directory)) { + for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitOperationExecutorTest.java b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitOperationExecutorTest.java new file mode 100644 index 00000000..e6be4729 --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitOperationExecutorTest.java @@ -0,0 +1,109 @@ +package tech.easyflow.skill.gitimport; + +import org.junit.After; +import org.junit.Test; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +/** + * {@link SkillGitOperationExecutor} 并发与总超时契约测试。 + */ +public class SkillGitOperationExecutorTest { + + private SkillGitOperationExecutor operationExecutor; + + /** + * 关闭测试中创建的远程操作执行器。 + */ + @After + public void tearDown() { + if (operationExecutor != null) { + operationExecutor.close(); + } + } + + /** + * 单实例并发达到上限时快速拒绝额外任务。 + * + * @throws Exception 线程协作失败 + */ + @Test + public void rejectsOperationWhenConcurrencyIsExhausted() throws Exception { + SkillGitImportProperties properties = new SkillGitImportProperties(); + properties.setMaxConcurrentOperations(1); + operationExecutor = new SkillGitOperationExecutor(properties); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future first = caller.submit(() -> operationExecutor.execute(() -> { + started.countDown(); + release.await(); + return "done"; + })); + assertTrue(started.await(2, TimeUnit.SECONDS)); + + BusinessException exception = assertThrows(BusinessException.class, + () -> operationExecutor.execute(() -> "second")); + + assertEquals(429, exception.getHttpStatus()); + release.countDown(); + assertEquals("done", first.get(2, TimeUnit.SECONDS)); + } finally { + release.countDown(); + caller.shutdownNow(); + } + } + + /** + * 超过远程操作总时限时返回网关超时并取消任务。 + */ + @Test + public void cancelsOperationAfterTotalTimeout() { + SkillGitImportProperties properties = new SkillGitImportProperties(); + properties.setOperationTimeout(Duration.ofMillis(30)); + operationExecutor = new SkillGitOperationExecutor(properties); + + BusinessException exception = assertThrows(BusinessException.class, + () -> operationExecutor.execute(() -> { + Thread.sleep(5_000L); + return "late"; + })); + + assertEquals(504, exception.getHttpStatus()); + } + + /** + * 任务忽略中断并在超时后返回资源时，执行器负责回收该结果。 + * + * @throws Exception 等待清理回调失败 + */ + @Test + public void cleansResultProducedAfterTimeout() throws Exception { + SkillGitImportProperties properties = new SkillGitImportProperties(); + properties.setOperationTimeout(Duration.ofMillis(30)); + operationExecutor = new SkillGitOperationExecutor(properties); + CountDownLatch cleaned = new CountDownLatch(1); + + assertThrows(BusinessException.class, () -> operationExecutor.execute(() -> { + try { + Thread.sleep(5_000L); + } catch (InterruptedException ignored) { + // 模拟底层驱动在取消后仍返回已经创建的资源。 + } + return "late-resource"; + }, result -> cleaned.countDown())); + + assertTrue(cleaned.await(2, TimeUnit.SECONDS)); + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitScanRateLimiterTest.java b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitScanRateLimiterTest.java new file mode 100644 index 00000000..8e5953cf --- /dev/null +++ b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/gitimport/SkillGitScanRateLimiterTest.java @@ -0,0 +1,63 @@ +package tech.easyflow.skill.gitimport; + +import com.alicp.jetcache.Cache; +import com.alicp.jetcache.CacheResult; +import org.junit.Test; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.math.BigInteger; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * {@link SkillGitScanRateLimiter} 分布式频控契约测试。 + */ +public class SkillGitScanRateLimiterTest { + + /** + * 首次扫描写入带过期时间的账号频控键。 + */ + @Test + public void acceptsFirstScanAndStoresAccountScopedKey() { + Cache cache = mock(Cache.class); + when(cache.PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE), + anyLong(), eq(TimeUnit.MILLISECONDS))).thenReturn(CacheResult.SUCCESS_WITHOUT_MSG); + SkillGitScanRateLimiter limiter = new SkillGitScanRateLimiter(cache, new SkillGitImportProperties()); + + limiter.check(account()); + + verify(cache).PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE), + eq(2_000L), eq(TimeUnit.MILLISECONDS)); + } + + /** + * 频控键已存在时返回 HTTP 429。 + */ + @Test + public void rejectsRepeatedScanWithinMinimumInterval() { + Cache cache = mock(Cache.class); + when(cache.PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE), + anyLong(), eq(TimeUnit.MILLISECONDS))).thenReturn(CacheResult.EXISTS_WITHOUT_MSG); + SkillGitScanRateLimiter limiter = new SkillGitScanRateLimiter(cache, new SkillGitImportProperties()); + + BusinessException exception = assertThrows(BusinessException.class, + () -> limiter.check(account())); + + assertEquals(429, exception.getHttpStatus()); + } + + private LoginAccount account() { + LoginAccount account = new LoginAccount(); + account.setTenantId(BigInteger.TEN); + account.setId(BigInteger.valueOf(20)); + return account; + } +} diff --git a/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/imports/StandardSkillPackageContractTest.java b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/imports/StandardSkillPackageContractTest.java index ed261826..3ed9363b 100644 --- a/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/imports/StandardSkillPackageContractTest.java +++ b/easyflow-modules/easyflow-module-skill/src/test/java/tech/easyflow/skill/imports/StandardSkillPackageContractTest.java @@ -9,17 +9,24 @@ import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.satoken.util.SaTokenUtil; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.skill.entity.Skill; +import tech.easyflow.skill.entity.SkillImportStage; import tech.easyflow.skill.service.SkillService; import tech.easyflow.skill.store.DBSkillContentStore; import tech.easyflow.system.service.ResourceAccessService; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -29,8 +36,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -60,22 +70,66 @@ public class StandardSkillPackageContractTest { } /** - * 一个上传 ZIP 只能承载一个 Skill，批量导入由多个 token 独立完成。 + * 一个父目录包装的多 Skill ZIP 会被拆成多个独立 token 预览。 */ @Test - public void previewRejectsMultipleSkillsInOneZip() { - SkillService skillService = mock(SkillService.class); - when(skillService.list(any(QueryWrapper.class))).thenReturn(List.of()); - SkillImportServiceImpl service = importService(skillService); + public void previewSplitsParentWrappedSkillsIntoIndependentTokens() { + BatchFixture fixture = batchImportService(); + MultipartFile file = multipartFile("skill-bundle.zip", multiSkillZip("skill-bundle/", 2)); - SkillImportPreview preview; + List previews; try (MockedStatic ignored = login()) { - preview = service.previewStandardForTest(new ByteArrayInputStream(multiSkillZip())); + previews = fixture.service().previewBatch(List.of(file)); } - assertTrue(preview.getSkills().isEmpty()); - assertTrue(preview.getIssues().stream() - .anyMatch(issue -> "STANDARD_PACKAGE_SKILL_COUNT".equals(issue.getCode()))); + assertEquals(2, previews.size()); + assertEquals(List.of("skill-1", "skill-2"), previews.stream() + .map(preview -> preview.getSkills().get(0).getName()).toList()); + assertTrue(previews.stream().allMatch(preview -> preview.getSkills().size() == 1)); + assertTrue(previews.stream().allMatch(preview -> preview.getImportToken() != null)); + assertTrue(previews.stream().allMatch(preview -> "skill-bundle.zip".equals(preview.getSourceName()))); + verify(fixture.fileStorageService(), times(2)).save(any(java.io.File.class), anyString()); + verify(fixture.stageStore(), times(2)).create(anyString(), anyString()); + } + + /** + * 单个 ZIP 解析出的 Skill 超过统一上限时，在创建暂存 token 前拒绝。 + */ + @Test + public void previewRejectsMoreThanFiftySkills() { + BatchFixture fixture = batchImportService(); + MultipartFile file = multipartFile("too-many.zip", multiSkillZip("", 51)); + + BusinessException exception; + try (MockedStatic ignored = login()) { + exception = assertThrows(BusinessException.class, + () -> fixture.service().previewBatch(List.of(file))); + } + + assertTrue(exception.getMessage().contains("50")); + verify(fixture.fileStorageService(), times(0)).save(any(java.io.File.class), anyString()); + verify(fixture.stageStore(), times(0)).create(anyString(), anyString()); + } + + /** + * 多个上传包合计超过五十个 Skill 时，已创建的独立 token 会全部回滚。 + */ + @Test + public void previewEnforcesFiftySkillLimitAcrossUploadedPackages() { + BatchFixture fixture = batchImportService(); + MultipartFile first = multipartFile("first.zip", multiSkillZip("", 25)); + MultipartFile second = multipartFile("second.zip", multiSkillZip("bundle/", 26)); + + BusinessException exception; + try (MockedStatic ignored = login()) { + exception = assertThrows(BusinessException.class, + () -> fixture.service().previewBatch(List.of(first, second))); + } + + assertTrue(exception.getMessage().contains("50")); + verify(fixture.fileStorageService(), times(25)).save(any(java.io.File.class), anyString()); + verify(fixture.stageStore(), times(25)).create(anyString(), anyString()); + verify(fixture.stageStore(), times(25)).cancel(anyString()); } /** @@ -151,12 +205,21 @@ public class StandardSkillPackageContractTest { } } - private byte[] multiSkillZip() { + /** + * 创建包含指定数量 Skill 的测试 ZIP。 + * + * @param parent 可选父目录前缀 + * @param count Skill 数量 + * @return ZIP 字节 + */ + private byte[] multiSkillZip(String parent, int count) { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ZipOutputStream zip = new ZipOutputStream(bytes, StandardCharsets.UTF_8)) { - write(zip, "alpha-skill/SKILL.md", skillContent("alpha-skill")); - write(zip, "beta-skill/SKILL.md", skillContent("beta-skill")); + for (int index = 1; index <= count; index++) { + String name = "skill-" + index; + write(zip, parent + name + "/SKILL.md", skillContent(name)); + } } return bytes.toByteArray(); } catch (Exception exception) { @@ -164,6 +227,64 @@ public class StandardSkillPackageContractTest { } } + /** + * 创建带内存文件存储行为的批量导入测试夹具。 + * + * @return 批量导入测试夹具 + */ + private BatchFixture batchImportService() { + SkillService skillService = mock(SkillService.class); + when(skillService.list(any(QueryWrapper.class))).thenReturn(List.of()); + FileStorageService fileStorageService = mock(FileStorageService.class); + SkillImportStageStore stageStore = mock(SkillImportStageStore.class); + Map storedFiles = new HashMap<>(); + AtomicInteger sequence = new AtomicInteger(); + try { + when(fileStorageService.save(any(java.io.File.class), anyString())).thenAnswer(invocation -> { + java.io.File file = invocation.getArgument(0); + String path = "stored-" + sequence.incrementAndGet() + ".zip"; + storedFiles.put(path, Files.readAllBytes(file.toPath())); + return path; + }); + when(fileStorageService.readStream(anyString())).thenAnswer(invocation -> + new ByteArrayInputStream(storedFiles.get(invocation.getArgument(0)))); + } catch (IOException exception) { + throw new IllegalStateException("创建测试文件存储失败", exception); + } + when(stageStore.create(anyString(), anyString())).thenAnswer(invocation -> { + SkillImportStage stage = new SkillImportStage(); + stage.setFilePath(invocation.getArgument(0)); + stage.setOriginalName(invocation.getArgument(1)); + stage.setImportToken("token-" + sequence.get()); + stage.setExpiresAt(new Date(System.currentTimeMillis() + 60_000)); + return stage; + }); + SkillImportServiceImpl service = new SkillImportServiceImpl(skillService, + mock(DBSkillContentStore.class), fileStorageService, stageStore, + mock(ResourceAccessService.class)); + return new BatchFixture(service, fileStorageService, stageStore); + } + + /** + * 创建可重复打开输入流的上传文件替身。 + * + * @param name 原始文件名 + * @param bytes 文件字节 + * @return 上传文件替身 + */ + private MultipartFile multipartFile(String name, byte[] bytes) { + MultipartFile file = mock(MultipartFile.class); + when(file.isEmpty()).thenReturn(false); + when(file.getSize()).thenReturn((long) bytes.length); + when(file.getOriginalFilename()).thenReturn(name); + try { + when(file.getInputStream()).thenAnswer(ignored -> new ByteArrayInputStream(bytes)); + } catch (IOException exception) { + throw new IllegalStateException("创建测试上传文件失败", exception); + } + return file; + } + private void write(ZipOutputStream zip, String path, String content) throws Exception { zip.putNextEntry(new ZipEntry(path)); zip.write(content.getBytes(StandardCharsets.UTF_8)); @@ -189,4 +310,16 @@ public class StandardSkillPackageContractTest { throw new IllegalStateException("读取标准 Skill 测试包失败", exception); } } + + /** + * 批量预检测试依赖集合。 + * + * @param service 导入服务 + * @param fileStorageService 文件存储 + * @param stageStore 导入暂存仓库 + */ + private record BatchFixture(SkillImportServiceImpl service, + FileStorageService fileStorageService, + SkillImportStageStore stageStore) { + } } diff --git a/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.behavior.test.ts b/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.behavior.test.ts new file mode 100644 index 00000000..97e2bcc5 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.behavior.test.ts @@ -0,0 +1,218 @@ +import { flushPromises, mount } from '@vue/test-utils'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import SkillCreateDialog from './SkillCreateDialog.vue'; + +const apiMocks = vi.hoisted(() => ({ + cancelSkillImport: vi.fn(), + importSkillConfirmBatch: vi.fn(), + importSkillPreviews: vi.fn(), + prepareSkillGitCandidates: vi.fn(), + saveSkill: vi.fn(), + scanSkillGitRepository: vi.fn(), +})); + +vi.mock('./api', () => apiMocks); + +describe('skill Git repository import dialog', () => { + beforeEach(() => { + vi.clearAllMocks(); + apiMocks.scanSkillGitRepository.mockResolvedValue({ + data: { + candidates: [ + { + candidateId: 'a'.repeat(64), + description: '聚合热门 AI 内容', + issues: [], + name: 'aihot', + path: 'aihot', + resourceCount: 2, + status: 'IMPORTABLE', + totalBytes: 2048, + }, + ], + expiresAt: '2026-08-15T14:00:00Z', + repository: { + commitSha: 'b'.repeat(40), + defaultBranch: 'main', + name: 'khazix-skills', + url: 'https://github.com/KKKKhazix/khazix-skills', + }, + scanToken: 'c'.repeat(32), + summary: { discovered: 1, importable: 1, needsAttention: 0 }, + }, + errorCode: 0, + }); + apiMocks.prepareSkillGitCandidates.mockResolvedValue({ + data: [ + { + importToken: 'd'.repeat(32), + issues: [], + skills: [ + { + conflict: false, + name: 'aihot', + packageId: 'aihot', + packageRoot: 'aihot', + }, + ], + }, + ], + errorCode: 0, + }); + apiMocks.importSkillConfirmBatch.mockResolvedValue({ + data: [ + { + importToken: 'd'.repeat(32), + skills: [{ id: 101, name: 'aihot' }], + success: true, + }, + ], + errorCode: 0, + }); + }); + + it('scans, selects and imports one repository candidate through the standard pipeline', async () => { + const wrapper = mount(SkillCreateDialog, { + global: { + stubs: { + ElDialog: { + props: ['modelValue', 'title'], + template: + '

{{ title }}

', + }, + }, + }, + props: { + categories: [], + defaultMode: 'import', + modelValue: true, + }, + }); + await flushPromises(); + + const sourceButtons = wrapper.findAllComponents({ name: 'ElRadioButton' }); + sourceButtons + .find((item) => item.props('value') === 'git') + ?.vm.$emit('click'); + sourceButtons + .find((item) => item.props('value') === 'git') + ?.vm.$emit('update:modelValue', 'git'); + const sourceGroup = wrapper.findAllComponents({ name: 'ElRadioGroup' })[1]; + sourceGroup?.vm.$emit('update:modelValue', 'git'); + await flushPromises(); + + const repositoryInput = wrapper.find( + 'input[placeholder="输入 HTTPS Git 仓库地址"]', + ); + await repositoryInput.setValue( + 'https://github.com/KKKKhazix/khazix-skills', + ); + const scanButton = wrapper + .findAll('button') + .find((button) => button.text().trim() === '扫描仓库'); + await scanButton?.trigger('click'); + await flushPromises(); + + expect(apiMocks.scanSkillGitRepository).toHaveBeenCalledWith( + 'https://github.com/KKKKhazix/khazix-skills', + ); + expect(wrapper.text()).toContain('khazix-skills'); + expect(wrapper.text()).toContain('aihot'); + + await wrapper + .get('.skill-create-dialog__candidate-toggle') + .trigger('click'); + const importButton = wrapper + .findAll('button') + .find((button) => button.text().includes('导入所选')); + await importButton?.trigger('click'); + await flushPromises(); + + expect(apiMocks.prepareSkillGitCandidates).toHaveBeenCalledWith( + 'c'.repeat(32), + ['a'.repeat(64)], + ); + expect(apiMocks.importSkillConfirmBatch).toHaveBeenCalledWith([ + expect.objectContaining({ + conflictStrategy: 'REJECT', + importToken: 'd'.repeat(32), + visibilityScope: 'PRIVATE', + }), + ]); + expect(wrapper.emitted('imported')).toEqual([[1]]); + }); + + it('renders and confirms every Skill returned from one multi-Skill ZIP', async () => { + const archive = new File(['fixture'], 'skill-bundle.zip', { + type: 'application/zip', + }); + apiMocks.importSkillPreviews.mockResolvedValue({ + data: ['skill-1', 'skill-2'].map((name, index) => ({ + importToken: `${index + 1}`.repeat(32), + issues: [], + skills: [ + { + conflict: false, + name, + packageId: name, + packageRoot: name, + }, + ], + sourceName: archive.name, + })), + errorCode: 0, + }); + apiMocks.importSkillConfirmBatch.mockResolvedValue({ + data: ['skill-1', 'skill-2'].map((name, index) => ({ + importToken: `${index + 1}`.repeat(32), + skills: [{ id: index + 1, name }], + success: true, + })), + errorCode: 0, + }); + const wrapper = mount(SkillCreateDialog, { + global: { + stubs: { + ElDialog: { + props: ['modelValue', 'title'], + template: + '

{{ title }}

', + }, + }, + }, + props: { + categories: [], + defaultMode: 'import', + modelValue: true, + }, + }); + await flushPromises(); + + const fileInput = wrapper.get('input[type="file"]'); + Object.defineProperty(fileInput.element, 'files', { + configurable: true, + value: [archive], + }); + await fileInput.trigger('change'); + await flushPromises(); + + expect(apiMocks.importSkillPreviews).toHaveBeenCalledWith([archive]); + expect(wrapper.text()).toContain('skill-1'); + expect(wrapper.text()).toContain('skill-2'); + expect(wrapper.text()).toContain('skill-bundle.zip'); + + const importButton = wrapper + .findAll('button') + .find((button) => button.text().trim() === '导入'); + await importButton?.trigger('click'); + await flushPromises(); + + expect(apiMocks.importSkillConfirmBatch).toHaveBeenCalledWith([ + expect.objectContaining({ importToken: '1'.repeat(32) }), + expect.objectContaining({ importToken: '2'.repeat(32) }), + ]); + expect(wrapper.emitted('imported')).toEqual([[2]]); + }); +}); diff --git a/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.test.ts b/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.test.ts index 76d882ec..909d853d 100644 --- a/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.test.ts +++ b/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.test.ts @@ -8,7 +8,9 @@ describe('skill create and import contract', () => { expect(dialogSource).toContain('value="import">批量导入'); expect(dialogSource).toContain('accept=".zip,application/zip"'); expect(dialogSource).toContain('multiple'); - expect(dialogSource).toContain('单次最多导入 20 个技能'); + expect(dialogSource).toContain('const MAX_IMPORT_SKILLS = 50'); + expect(dialogSource).toContain('支持单技能或多技能 ZIP'); + expect(dialogSource).toContain('response.data.map((preview, index)'); expect(dialogSource).toContain('importSkillConfirmBatch'); expect(dialogSource).not.toContain('.efskill'); }); @@ -18,4 +20,14 @@ describe('skill create and import contract', () => { expect(dialogSource).toContain('v-model="form.visibilityScope"'); expect(dialogSource).toContain('visibilityScope: form.visibilityScope'); }); + + it('supports scanning arbitrary HTTPS Git repositories and selecting candidates', () => { + expect(dialogSource).toContain('value="git">Git 仓库'); + expect(dialogSource).toContain('scanSkillGitRepository'); + expect(dialogSource).toContain('prepareSkillGitCandidates'); + expect(dialogSource).toContain('搜索技能名称、用途或路径'); + expect(dialogSource).toContain('toggleGitCandidate(candidate)'); + expect(dialogSource).toContain('next.size >= MAX_IMPORT_SKILLS'); + expect(dialogSource).not.toContain('github.com/'); + }); }); diff --git a/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.vue b/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.vue index 3d29046d..c04f4a1d 100644 --- a/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.vue +++ b/easyflow-ui-admin/app/src/views/ai/skill/SkillCreateDialog.vue @@ -3,6 +3,8 @@ import type { FormInstance, FormRules } from 'element-plus'; import type { SkillCategory, + SkillGitCandidate, + SkillGitScanResult, SkillImportPreview, SkillInfo, SkillVisibilityScope, @@ -10,7 +12,14 @@ import type { import { computed, nextTick, reactive, ref, watch } from 'vue'; -import { Document, UploadFilled } from '@element-plus/icons-vue'; +import { + Check, + Document, + Link, + Plus, + Search, + UploadFilled, +} from '@element-plus/icons-vue'; import { ElButton, ElDialog, @@ -30,7 +39,9 @@ import { cancelSkillImport, importSkillConfirmBatch, importSkillPreviews, + prepareSkillGitCandidates, saveSkill, + scanSkillGitRepository, } from './api'; import { resolveSkillApiErrorMessage } from './skill-api-error'; import { flattenSkillCategories } from './skill-category'; @@ -38,9 +49,11 @@ import { buildInitialSkillDraft } from './skill-create'; type CreateMode = 'import' | 'manual'; type ConflictStrategy = 'OVERWRITE' | 'REJECT' | 'RENAME'; +type ImportSource = 'git' | 'zip'; interface ImportRow { - file: File; + key: string; + label: string; message?: string; preview?: SkillImportPreview; rename: string; @@ -69,14 +82,21 @@ const emit = defineEmits<{ 'update:modelValue': [value: boolean]; }>(); +const MAX_IMPORT_SKILLS = 50; + const formRef = ref(); const nameInputRef = ref>(); const fileInputRef = ref(); const mode = ref('manual'); +const importSource = ref('zip'); const busy = ref(false); const actionError = ref(''); const importRows = ref([]); const importFinished = ref(false); +const gitRepositoryUrl = ref(''); +const gitSearch = ref(''); +const gitScan = ref(); +const selectedGitCandidateIds = ref>(new Set()); const form = reactive({ categoryId: '' as number | string, displayName: '', @@ -102,28 +122,44 @@ const rules: FormRules = { { max: 128, message: '技能名称不能超过 128 个字符', trigger: 'blur' }, ], }; -const confirmText = computed(() => - mode.value === 'manual' ? '创建并进入编辑' : '导入', +const confirmText = computed(() => { + if (mode.value === 'manual') return '创建并进入编辑'; + if (importSource.value !== 'git' || importRows.value.length > 0) + return '导入'; + const count = selectedGitCandidateIds.value.size; + return `导入所选${count > 0 ? `（${count}）` : ''}`; +}); +const dialogWidth = computed(() => + mode.value === 'import' && importSource.value === 'git' + ? 'min(920px, calc(100vw - 32px))' + : 'min(720px, calc(100vw - 32px))', ); const importReady = computed( - () => - importRows.value.length > 0 && - importRows.value.every((row) => { - const item = row.preview?.skills[0]; - if (!row.preview?.importToken || !item || importErrorCount(row.preview)) - return false; - if (!item.conflict) return true; - if (row.strategy === 'OVERWRITE') return item.overwriteAllowed === true; - if (row.strategy === 'RENAME') return canonicalName(row.rename); - return false; - }), + () => importRows.value.length > 0 && rowsReady(importRows.value), ); +const filteredGitCandidates = computed(() => { + const keyword = normalizeSearch(gitSearch.value); + if (!keyword) return gitScan.value?.candidates || []; + return (gitScan.value?.candidates || []).filter((candidate) => + normalizeSearch( + `${candidate.name} ${candidate.description || ''} ${candidate.path}`, + ).includes(keyword), + ); +}); +const submitDisabled = computed(() => { + if (mode.value === 'manual') return false; + if (importRows.value.length > 0) return !importReady.value; + return ( + importSource.value === 'zip' || selectedGitCandidateIds.value.size === 0 + ); +}); watch( () => props.modelValue, (open) => { if (!open) return; mode.value = props.defaultMode; + importSource.value = 'zip'; Object.assign(form, { categoryId: props.defaultCategoryId || '', displayName: '', @@ -131,6 +167,10 @@ watch( }); importRows.value = []; importFinished.value = false; + gitRepositoryUrl.value = ''; + gitSearch.value = ''; + gitScan.value = undefined; + selectedGitCandidateIds.value = new Set(); actionError.value = ''; formRef.value?.clearValidate(); if (mode.value === 'manual') @@ -141,7 +181,13 @@ watch( async function submit() { if (busy.value) return; - await (mode.value === 'manual' ? createManualSkill() : confirmImports()); + if (mode.value === 'manual') { + await createManualSkill(); + } else if (importSource.value === 'git' && importRows.value.length === 0) { + await prepareGitImports(); + } else { + await confirmImports(); + } } async function createManualSkill() { @@ -185,8 +231,8 @@ async function handleDrop(event: DragEvent) { async function previewFiles(files: File[]) { if (busy.value || files.length === 0) return; - if (files.length > 20) { - ElMessage.warning('单次最多导入 20 个技能'); + if (files.length > MAX_IMPORT_SKILLS) { + ElMessage.warning(`单次最多选择 ${MAX_IMPORT_SKILLS} 个 ZIP`); return; } if (files.some((file) => !file.name.toLowerCase().endsWith('.zip'))) { @@ -202,11 +248,14 @@ async function previewFiles(files: File[]) { actionError.value = response.message || '导入预检失败'; return; } - importRows.value = files.map((file, index) => { - const preview = response.data[index]; + importRows.value = response.data.map((preview, index) => { const item = preview?.skills[0]; + const sourceName = preview?.sourceName || `导入包 ${index + 1}`; return { - file, + key: + preview?.importToken || + `${sourceName}-${item?.packageRoot || item?.packageId || 'invalid'}-${index}`, + label: sourceName, preview, rename: item ? `${item.name}-copy` : '', strategy: item?.conflict ? 'RENAME' : 'REJECT', @@ -227,11 +276,15 @@ async function confirmImports() { ElMessage.warning('请先处理导入错误和名称冲突'); return; } + await confirmImportRows(importRows.value); +} + +async function confirmImportRows(rows: ImportRow[]) { busy.value = true; actionError.value = ''; try { const response = await importSkillConfirmBatch( - importRows.value.map((row) => { + rows.map((row) => { const item = row.preview!.skills[0]!; return { categoryId: form.categoryId || undefined, @@ -251,7 +304,7 @@ async function confirmImports() { } let succeeded = 0; response.data.forEach((result, index) => { - const row = importRows.value[index]; + const row = rows[index]; if (!row) return; row.result = result.success ? 'success' : 'failed'; row.message = result.message; @@ -259,7 +312,7 @@ async function confirmImports() { }); importFinished.value = true; emit('imported', succeeded); - if (succeeded === importRows.value.length) { + if (succeeded === rows.length) { ElMessage.success(`已导入 ${succeeded} 个技能`); emit('update:modelValue', false); } @@ -270,6 +323,101 @@ async function confirmImports() { } } +async function scanGitRepo() { + if (busy.value) return; + const repositoryUrl = gitRepositoryUrl.value.trim(); + if (!repositoryUrl) { + ElMessage.warning('请输入 Git 仓库地址'); + return; + } + await cancelPendingImports(); + busy.value = true; + actionError.value = ''; + importRows.value = []; + importFinished.value = false; + selectedGitCandidateIds.value = new Set(); + try { + const response = await scanSkillGitRepository(repositoryUrl); + if (response.errorCode !== 0 || !response.data?.scanToken) { + actionError.value = response.message || '仓库扫描失败'; + gitScan.value = undefined; + return; + } + gitScan.value = response.data; + if (response.data.summary.discovered === 0) { + ElMessage.info('仓库中未发现 SKILL.md'); + } + } catch (error) { + gitScan.value = undefined; + actionError.value = resolveSkillApiErrorMessage( + error, + '仓库扫描失败，请检查地址后重试', + ); + } finally { + busy.value = false; + } +} + +function toggleGitCandidate(candidate: SkillGitCandidate) { + if (candidate.status !== 'IMPORTABLE' || busy.value) return; + const next = new Set(selectedGitCandidateIds.value); + if (next.has(candidate.candidateId)) { + next.delete(candidate.candidateId); + } else if (next.size >= MAX_IMPORT_SKILLS) { + ElMessage.warning(`单次最多导入 ${MAX_IMPORT_SKILLS} 个技能`); + return; + } else { + next.add(candidate.candidateId); + } + selectedGitCandidateIds.value = next; +} + +async function prepareGitImports() { + const scan = gitScan.value; + const selected = scan?.candidates.filter((candidate) => + selectedGitCandidateIds.value.has(candidate.candidateId), + ); + if (!scan || !selected?.length) { + ElMessage.warning('请先扫描并选择要导入的技能'); + return; + } + busy.value = true; + actionError.value = ''; + try { + const response = await prepareSkillGitCandidates( + scan.scanToken, + selected.map((candidate) => candidate.candidateId), + ); + if (response.errorCode !== 0) { + actionError.value = response.message || '准备导入失败'; + return; + } + const rows = selected.map((candidate, index): ImportRow => { + const preview = response.data[index]; + const item = preview?.skills[0]; + return { + key: candidate.candidateId, + label: candidate.path, + preview, + rename: item ? `${item.name}-copy` : '', + strategy: item?.conflict ? 'RENAME' : 'REJECT', + }; + }); + importRows.value = rows; + if (rowsReady(rows)) { + busy.value = false; + await confirmImportRows(rows); + } + } catch (error) { + actionError.value = resolveSkillApiErrorMessage( + error, + '准备导入失败，请重新扫描', + ); + } finally { + busy.value = false; + } +} + async function requestClose() { if (busy.value) return; await cancelPendingImports(); @@ -291,6 +439,28 @@ function importErrorCount(preview?: SkillImportPreview) { .length; } +function rowsReady(rows: ImportRow[]) { + return rows.every((row) => { + const item = row.preview?.skills[0]; + if (!row.preview?.importToken || !item || importErrorCount(row.preview)) + return false; + if (!item.conflict) return true; + if (row.strategy === 'OVERWRITE') return item.overwriteAllowed === true; + if (row.strategy === 'RENAME') return canonicalName(row.rename); + return false; + }); +} + +function normalizeSearch(value: string) { + return value.trim().toLocaleLowerCase().replaceAll(/\s+/g, ' '); +} + +function formatBytes(bytes: number) { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${Math.ceil(bytes / 1024)} KB`; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + function canonicalName(value: string) { return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value.trim()); } @@ -304,7 +474,7 @@ function scopeLabel(scope: SkillVisibilityScope) {