feat: 支持 Skill 仓库扫描与批量导入

- 统一支持本地单包、多包和 Git 候选最多 50 项导入

- 固定提交并限制 SSRF、DNS、并发、频率、超时和仓库资源

- 增加导入对话框、接口契约和安全回归测试
This commit is contained in:
2026-08-19 22:39:55 +08:00
parent 4e8640dcaf
commit 9579619384
28 changed files with 3949 additions and 86 deletions

View File

@@ -0,0 +1,103 @@
package tech.easyflow.skill.gitimport;
import org.junit.Before;
import org.junit.Test;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URL;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
/**
* {@link GitRepositoryAccessPolicy} 地址安全契约测试。
*/
public class GitRepositoryAccessPolicyTest {
private GitRepositoryAccessPolicy policy;
/**
* 初始化不依赖公网 DNS 的可信测试主机。
*/
@Before
public void setUp() {
SkillGitImportProperties properties = new SkillGitImportProperties();
properties.setTrustedPrivateHosts(Set.of("git.example.test"));
policy = new GitRepositoryAccessPolicy(properties);
}
/**
* 仓库地址允许省略 .git 后缀并移除结尾斜线。
*/
@Test
public void acceptsHttpsRepositoryWithOptionalGitSuffix() {
assertEquals(policy.normalizeRepositoryUrl(
" https://git.example.test/team/skills/ "),
"https://git.example.test/team/skills");
assertEquals(policy.normalizeRepositoryUrl(
"https://git.example.test/team/skills.git"),
"https://git.example.test/team/skills.git");
assertEquals(policy.normalizeRepositoryUrl("https://192.0.1.1/team/skills.git"),
"https://192.0.1.1/team/skills.git");
}
/**
* 拒绝非 HTTPS、内嵌凭证和 URL 参数。
*/
@Test
public void rejectsUnsafeRepositoryInputs() {
assertRejected("http://git.example.test/team/skills.git");
assertRejected("https://user:secret@git.example.test/team/skills.git");
assertRejected("https://git.example.test/team/skills.git?token=secret");
assertRejected("https://git.example.test:8443/team/skills.git");
assertRejected("https://127.0.0.1/team/skills.git");
}
/**
* 实际连接使用策略校验后返回的固定 DNS 地址。
*
* @throws Exception 构造测试地址失败
*/
@Test
public void resolvesAddressesForPinnedConnection() throws Exception {
SkillGitImportProperties properties = new SkillGitImportProperties();
InetAddress publicAddress = InetAddress.getByAddress(new byte[]{93, (byte) 184, (byte) 216, 34});
GitRepositoryAccessPolicy pinnedPolicy = new GitRepositoryAccessPolicy(
properties, host -> new InetAddress[]{publicAddress});
GitRepositoryAccessPolicy.ResolvedConnection resolved = pinnedPolicy.resolveConnection(
new URL("https://git.example.test/team/skills.git/info/refs"));
assertEquals("git.example.test", resolved.host());
assertEquals(publicAddress, resolved.addresses()[0]);
}
/**
* 输入校验后主机重新绑定到内网时,实际连接阶段必须拒绝。
*
* @throws Exception 构造测试地址失败
*/
@Test
public void rejectsDnsRebindingBeforeConnection() throws Exception {
SkillGitImportProperties properties = new SkillGitImportProperties();
InetAddress publicAddress = InetAddress.getByAddress(new byte[]{93, (byte) 184, (byte) 216, 34});
InetAddress loopback = InetAddress.getByAddress(new byte[]{127, 0, 0, 1});
AtomicInteger resolutions = new AtomicInteger();
GitRepositoryAccessPolicy rebindingPolicy = new GitRepositoryAccessPolicy(properties,
host -> resolutions.getAndIncrement() == 0
? new InetAddress[]{publicAddress} : new InetAddress[]{loopback});
assertEquals("https://git.example.test/team/skills.git",
rebindingPolicy.normalizeRepositoryUrl("https://git.example.test/team/skills.git"));
assertThrows(IOException.class, () -> rebindingPolicy.resolveConnection(
new URL("https://git.example.test/team/skills.git/info/refs")));
}
private void assertRejected(String value) {
assertThrows(BusinessException.class, () -> policy.normalizeRepositoryUrl(value));
}
}

View File

@@ -0,0 +1,104 @@
package tech.easyflow.skill.gitimport;
import org.eclipse.jgit.api.Git;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import java.util.zip.ZipInputStream;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
* {@link SkillGitCandidateScanner} 固定提交扫描与标准包组装测试。
*/
public class SkillGitCandidateScannerTest {
/**
* 发现根目录和嵌套 Skill并把不可解析的 LFS 资源标为需处理。
*
* @throws Exception 仓库或 ZIP 读写失败
*/
@Test
public void discoversCandidatesAndCreatesIndependentStandardZip() throws Exception {
SkillGitImportProperties properties = new SkillGitImportProperties();
SkillGitCandidateScanner scanner = new SkillGitCandidateScanner(properties);
Path directory = Files.createTempDirectory("skill-git-scanner-test-");
try (Git git = Git.init().setDirectory(directory.toFile()).call()) {
write(directory.resolve("SKILL.md"), skill("root-skill", "根目录技能"));
write(directory.resolve("references/guide.md"), "# Guide\n");
write(directory.resolve("nested-skill/SKILL.md"), skill("nested-skill", "嵌套技能"));
write(directory.resolve("nested-skill/assets/model.bin"),
"version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 10\n");
git.add().addFilepattern(".").call();
String commitSha = git.commit().setMessage("test")
.setAuthor("EasyFlow", "test@easyflow.local")
.setCommitter("EasyFlow", "test@easyflow.local").call().getName();
SkillGitRepositoryReader.Snapshot snapshot = new SkillGitRepositoryReader.Snapshot(
git.getRepository(), directory, "refs/heads/master", commitSha);
SkillGitRepositoryScan scan = scanner.scan(snapshot);
assertEquals(scan.candidates().size(), 2);
SkillGitScanResult.Candidate root = candidate(scan, "root-skill");
SkillGitScanResult.Candidate nested = candidate(scan, "nested-skill");
assertTrue(root.importable());
assertFalse(nested.importable());
assertTrue(nested.issues().stream()
.anyMatch(issue -> "GIT_LFS_UNRESOLVED".equals(issue.code())));
Path archive = scanner.createStandardZip(snapshot, scan, root.candidateId());
try {
assertEquals(zipEntries(archive),
Set.of("root-skill/SKILL.md", "root-skill/references/guide.md"));
} finally {
scanner.deleteArchive(archive);
}
// Snapshot 与 Git 共享 Repository交给 Git 的 try-with-resources 统一关闭。
} finally {
deleteDirectory(directory);
}
}
private SkillGitScanResult.Candidate candidate(SkillGitRepositoryScan scan, String name) {
return scan.candidates().stream().filter(item -> name.equals(item.name())).findFirst()
.orElseThrow();
}
private Set<String> zipEntries(Path archive) throws IOException {
Set<String> entries = new HashSet<>();
try (ZipInputStream input = new ZipInputStream(Files.newInputStream(archive))) {
java.util.zip.ZipEntry entry;
while ((entry = input.getNextEntry()) != null) {
entries.add(entry.getName());
}
}
return entries;
}
private String skill(String name, String description) {
return "---\nname: " + name + "\ndescription: " + description + "\n---\n# " + name + "\n";
}
private void write(Path path, String content) throws IOException {
Files.createDirectories(path.getParent());
Files.writeString(path, content, StandardCharsets.UTF_8);
}
private void deleteDirectory(Path directory) throws IOException {
if (!Files.exists(directory)) {
return;
}
try (java.util.stream.Stream<Path> paths = Files.walk(directory)) {
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
Files.deleteIfExists(path);
}
}
}
}

View File

@@ -0,0 +1,109 @@
package tech.easyflow.skill.gitimport;
import org.junit.After;
import org.junit.Test;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.time.Duration;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
/**
* {@link SkillGitOperationExecutor} 并发与总超时契约测试。
*/
public class SkillGitOperationExecutorTest {
private SkillGitOperationExecutor operationExecutor;
/**
* 关闭测试中创建的远程操作执行器。
*/
@After
public void tearDown() {
if (operationExecutor != null) {
operationExecutor.close();
}
}
/**
* 单实例并发达到上限时快速拒绝额外任务。
*
* @throws Exception 线程协作失败
*/
@Test
public void rejectsOperationWhenConcurrencyIsExhausted() throws Exception {
SkillGitImportProperties properties = new SkillGitImportProperties();
properties.setMaxConcurrentOperations(1);
operationExecutor = new SkillGitOperationExecutor(properties);
CountDownLatch started = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
ExecutorService caller = Executors.newSingleThreadExecutor();
try {
Future<String> first = caller.submit(() -> operationExecutor.execute(() -> {
started.countDown();
release.await();
return "done";
}));
assertTrue(started.await(2, TimeUnit.SECONDS));
BusinessException exception = assertThrows(BusinessException.class,
() -> operationExecutor.execute(() -> "second"));
assertEquals(429, exception.getHttpStatus());
release.countDown();
assertEquals("done", first.get(2, TimeUnit.SECONDS));
} finally {
release.countDown();
caller.shutdownNow();
}
}
/**
* 超过远程操作总时限时返回网关超时并取消任务。
*/
@Test
public void cancelsOperationAfterTotalTimeout() {
SkillGitImportProperties properties = new SkillGitImportProperties();
properties.setOperationTimeout(Duration.ofMillis(30));
operationExecutor = new SkillGitOperationExecutor(properties);
BusinessException exception = assertThrows(BusinessException.class,
() -> operationExecutor.execute(() -> {
Thread.sleep(5_000L);
return "late";
}));
assertEquals(504, exception.getHttpStatus());
}
/**
* 任务忽略中断并在超时后返回资源时,执行器负责回收该结果。
*
* @throws Exception 等待清理回调失败
*/
@Test
public void cleansResultProducedAfterTimeout() throws Exception {
SkillGitImportProperties properties = new SkillGitImportProperties();
properties.setOperationTimeout(Duration.ofMillis(30));
operationExecutor = new SkillGitOperationExecutor(properties);
CountDownLatch cleaned = new CountDownLatch(1);
assertThrows(BusinessException.class, () -> operationExecutor.execute(() -> {
try {
Thread.sleep(5_000L);
} catch (InterruptedException ignored) {
// 模拟底层驱动在取消后仍返回已经创建的资源。
}
return "late-resource";
}, result -> cleaned.countDown()));
assertTrue(cleaned.await(2, TimeUnit.SECONDS));
}
}

View File

@@ -0,0 +1,63 @@
package tech.easyflow.skill.gitimport;
import com.alicp.jetcache.Cache;
import com.alicp.jetcache.CacheResult;
import org.junit.Test;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.math.BigInteger;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* {@link SkillGitScanRateLimiter} 分布式频控契约测试。
*/
public class SkillGitScanRateLimiterTest {
/**
* 首次扫描写入带过期时间的账号频控键。
*/
@Test
public void acceptsFirstScanAndStoresAccountScopedKey() {
Cache<String, Object> cache = mock(Cache.class);
when(cache.PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE),
anyLong(), eq(TimeUnit.MILLISECONDS))).thenReturn(CacheResult.SUCCESS_WITHOUT_MSG);
SkillGitScanRateLimiter limiter = new SkillGitScanRateLimiter(cache, new SkillGitImportProperties());
limiter.check(account());
verify(cache).PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE),
eq(2_000L), eq(TimeUnit.MILLISECONDS));
}
/**
* 频控键已存在时返回 HTTP 429。
*/
@Test
public void rejectsRepeatedScanWithinMinimumInterval() {
Cache<String, Object> cache = mock(Cache.class);
when(cache.PUT_IF_ABSENT(eq("skill:git-scan:rate:10:20"), eq(Boolean.TRUE),
anyLong(), eq(TimeUnit.MILLISECONDS))).thenReturn(CacheResult.EXISTS_WITHOUT_MSG);
SkillGitScanRateLimiter limiter = new SkillGitScanRateLimiter(cache, new SkillGitImportProperties());
BusinessException exception = assertThrows(BusinessException.class,
() -> limiter.check(account()));
assertEquals(429, exception.getHttpStatus());
}
private LoginAccount account() {
LoginAccount account = new LoginAccount();
account.setTenantId(BigInteger.TEN);
account.setId(BigInteger.valueOf(20));
return account;
}
}

View File

@@ -9,17 +9,24 @@ import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.skill.entity.Skill;
import tech.easyflow.skill.entity.SkillImportStage;
import tech.easyflow.skill.service.SkillService;
import tech.easyflow.skill.store.DBSkillContentStore;
import tech.easyflow.system.service.ResourceAccessService;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -29,8 +36,11 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
@@ -60,22 +70,66 @@ public class StandardSkillPackageContractTest {
}
/**
* 一个上传 ZIP 只能承载一个 Skill批量导入由多个 token 独立完成
* 一个父目录包装的多 Skill ZIP 会被拆成多个独立 token 预览
*/
@Test
public void previewRejectsMultipleSkillsInOneZip() {
SkillService skillService = mock(SkillService.class);
when(skillService.list(any(QueryWrapper.class))).thenReturn(List.of());
SkillImportServiceImpl service = importService(skillService);
public void previewSplitsParentWrappedSkillsIntoIndependentTokens() {
BatchFixture fixture = batchImportService();
MultipartFile file = multipartFile("skill-bundle.zip", multiSkillZip("skill-bundle/", 2));
SkillImportPreview preview;
List<SkillImportPreview> previews;
try (MockedStatic<SaTokenUtil> ignored = login()) {
preview = service.previewStandardForTest(new ByteArrayInputStream(multiSkillZip()));
previews = fixture.service().previewBatch(List.of(file));
}
assertTrue(preview.getSkills().isEmpty());
assertTrue(preview.getIssues().stream()
.anyMatch(issue -> "STANDARD_PACKAGE_SKILL_COUNT".equals(issue.getCode())));
assertEquals(2, previews.size());
assertEquals(List.of("skill-1", "skill-2"), previews.stream()
.map(preview -> preview.getSkills().get(0).getName()).toList());
assertTrue(previews.stream().allMatch(preview -> preview.getSkills().size() == 1));
assertTrue(previews.stream().allMatch(preview -> preview.getImportToken() != null));
assertTrue(previews.stream().allMatch(preview -> "skill-bundle.zip".equals(preview.getSourceName())));
verify(fixture.fileStorageService(), times(2)).save(any(java.io.File.class), anyString());
verify(fixture.stageStore(), times(2)).create(anyString(), anyString());
}
/**
* 单个 ZIP 解析出的 Skill 超过统一上限时,在创建暂存 token 前拒绝。
*/
@Test
public void previewRejectsMoreThanFiftySkills() {
BatchFixture fixture = batchImportService();
MultipartFile file = multipartFile("too-many.zip", multiSkillZip("", 51));
BusinessException exception;
try (MockedStatic<SaTokenUtil> ignored = login()) {
exception = assertThrows(BusinessException.class,
() -> fixture.service().previewBatch(List.of(file)));
}
assertTrue(exception.getMessage().contains("50"));
verify(fixture.fileStorageService(), times(0)).save(any(java.io.File.class), anyString());
verify(fixture.stageStore(), times(0)).create(anyString(), anyString());
}
/**
* 多个上传包合计超过五十个 Skill 时,已创建的独立 token 会全部回滚。
*/
@Test
public void previewEnforcesFiftySkillLimitAcrossUploadedPackages() {
BatchFixture fixture = batchImportService();
MultipartFile first = multipartFile("first.zip", multiSkillZip("", 25));
MultipartFile second = multipartFile("second.zip", multiSkillZip("bundle/", 26));
BusinessException exception;
try (MockedStatic<SaTokenUtil> ignored = login()) {
exception = assertThrows(BusinessException.class,
() -> fixture.service().previewBatch(List.of(first, second)));
}
assertTrue(exception.getMessage().contains("50"));
verify(fixture.fileStorageService(), times(25)).save(any(java.io.File.class), anyString());
verify(fixture.stageStore(), times(25)).create(anyString(), anyString());
verify(fixture.stageStore(), times(25)).cancel(anyString());
}
/**
@@ -151,12 +205,21 @@ public class StandardSkillPackageContractTest {
}
}
private byte[] multiSkillZip() {
/**
* 创建包含指定数量 Skill 的测试 ZIP。
*
* @param parent 可选父目录前缀
* @param count Skill 数量
* @return ZIP 字节
*/
private byte[] multiSkillZip(String parent, int count) {
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (ZipOutputStream zip = new ZipOutputStream(bytes, StandardCharsets.UTF_8)) {
write(zip, "alpha-skill/SKILL.md", skillContent("alpha-skill"));
write(zip, "beta-skill/SKILL.md", skillContent("beta-skill"));
for (int index = 1; index <= count; index++) {
String name = "skill-" + index;
write(zip, parent + name + "/SKILL.md", skillContent(name));
}
}
return bytes.toByteArray();
} catch (Exception exception) {
@@ -164,6 +227,64 @@ public class StandardSkillPackageContractTest {
}
}
/**
* 创建带内存文件存储行为的批量导入测试夹具。
*
* @return 批量导入测试夹具
*/
private BatchFixture batchImportService() {
SkillService skillService = mock(SkillService.class);
when(skillService.list(any(QueryWrapper.class))).thenReturn(List.of());
FileStorageService fileStorageService = mock(FileStorageService.class);
SkillImportStageStore stageStore = mock(SkillImportStageStore.class);
Map<String, byte[]> storedFiles = new HashMap<>();
AtomicInteger sequence = new AtomicInteger();
try {
when(fileStorageService.save(any(java.io.File.class), anyString())).thenAnswer(invocation -> {
java.io.File file = invocation.getArgument(0);
String path = "stored-" + sequence.incrementAndGet() + ".zip";
storedFiles.put(path, Files.readAllBytes(file.toPath()));
return path;
});
when(fileStorageService.readStream(anyString())).thenAnswer(invocation ->
new ByteArrayInputStream(storedFiles.get(invocation.getArgument(0))));
} catch (IOException exception) {
throw new IllegalStateException("创建测试文件存储失败", exception);
}
when(stageStore.create(anyString(), anyString())).thenAnswer(invocation -> {
SkillImportStage stage = new SkillImportStage();
stage.setFilePath(invocation.getArgument(0));
stage.setOriginalName(invocation.getArgument(1));
stage.setImportToken("token-" + sequence.get());
stage.setExpiresAt(new Date(System.currentTimeMillis() + 60_000));
return stage;
});
SkillImportServiceImpl service = new SkillImportServiceImpl(skillService,
mock(DBSkillContentStore.class), fileStorageService, stageStore,
mock(ResourceAccessService.class));
return new BatchFixture(service, fileStorageService, stageStore);
}
/**
* 创建可重复打开输入流的上传文件替身。
*
* @param name 原始文件名
* @param bytes 文件字节
* @return 上传文件替身
*/
private MultipartFile multipartFile(String name, byte[] bytes) {
MultipartFile file = mock(MultipartFile.class);
when(file.isEmpty()).thenReturn(false);
when(file.getSize()).thenReturn((long) bytes.length);
when(file.getOriginalFilename()).thenReturn(name);
try {
when(file.getInputStream()).thenAnswer(ignored -> new ByteArrayInputStream(bytes));
} catch (IOException exception) {
throw new IllegalStateException("创建测试上传文件失败", exception);
}
return file;
}
private void write(ZipOutputStream zip, String path, String content) throws Exception {
zip.putNextEntry(new ZipEntry(path));
zip.write(content.getBytes(StandardCharsets.UTF_8));
@@ -189,4 +310,16 @@ public class StandardSkillPackageContractTest {
throw new IllegalStateException("读取标准 Skill 测试包失败", exception);
}
}
/**
* 批量预检测试依赖集合。
*
* @param service 导入服务
* @param fileStorageService 文件存储
* @param stageStore 导入暂存仓库
*/
private record BatchFixture(SkillImportServiceImpl service,
FileStorageService fileStorageService,
SkillImportStageStore stageStore) {
}
}