feat: 异步同步知识库分块检索索引
This commit is contained in:
@@ -1,23 +1,18 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
|
||||||
import tech.easyflow.ai.entity.Model;
|
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
|
||||||
import tech.easyflow.ai.service.ModelService;
|
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
|
||||||
import tech.easyflow.common.annotation.UsePermission;
|
import tech.easyflow.common.annotation.UsePermission;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
import tech.easyflow.common.web.controller.BaseController;
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||||
import com.easyagents.core.document.Document;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreOptions;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
@@ -28,12 +23,8 @@ import tech.easyflow.system.enums.ResourceAction;
|
|||||||
import tech.easyflow.system.enums.ResourceLookup;
|
import tech.easyflow.system.enums.ResourceLookup;
|
||||||
import tech.easyflow.system.permission.resource.RequireResourceAccess;
|
import tech.easyflow.system.permission.resource.RequireResourceAccess;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 控制层。
|
* 控制层。
|
||||||
@@ -44,19 +35,12 @@ import java.util.Map;
|
|||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/api/v1/documentChunk")
|
@RequestMapping("/api/v1/documentChunk")
|
||||||
@UsePermission(moduleName = "/api/v1/documentCollection")
|
@UsePermission(moduleName = "/api/v1/documentCollection")
|
||||||
public class DocumentChunkController extends BaseCurdController<DocumentChunkService, DocumentChunk> {
|
public class DocumentChunkController extends BaseController {
|
||||||
|
|
||||||
@Resource
|
private final DocumentChunkService documentChunkService;
|
||||||
DocumentCollectionService documentCollectionService;
|
|
||||||
|
|
||||||
@Resource
|
|
||||||
ModelService modelService;
|
|
||||||
|
|
||||||
@Resource
|
|
||||||
DocumentChunkService documentChunkService;
|
|
||||||
|
|
||||||
public DocumentChunkController(DocumentChunkService service) {
|
public DocumentChunkController(DocumentChunkService service) {
|
||||||
super(service);
|
this.documentChunkService = service;
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("page")
|
@GetMapping("page")
|
||||||
@@ -68,9 +52,30 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
|
|||||||
idExpr = "#request.getParameter('documentId')",
|
idExpr = "#request.getParameter('documentId')",
|
||||||
denyMessage = "无权限访问知识库"
|
denyMessage = "无权限访问知识库"
|
||||||
)
|
)
|
||||||
@Override
|
public Result<Page<DocumentChunk>> page(
|
||||||
public Result<Page<DocumentChunk>> page(HttpServletRequest request, String sortKey, String sortType, Long pageNumber, Long pageSize) {
|
HttpServletRequest request,
|
||||||
return super.page(request, sortKey, sortType, pageNumber, pageSize);
|
Long pageNumber,
|
||||||
|
Long pageSize
|
||||||
|
) {
|
||||||
|
String documentIdValue = request.getParameter("documentId");
|
||||||
|
if (documentIdValue == null || documentIdValue.isBlank()) {
|
||||||
|
return Result.<Page<DocumentChunk>>fail("documentId不能为空", null);
|
||||||
|
}
|
||||||
|
BigInteger documentId;
|
||||||
|
try {
|
||||||
|
documentId = new BigInteger(documentIdValue);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
return Result.<Page<DocumentChunk>>fail("documentId格式不正确", null);
|
||||||
|
}
|
||||||
|
long normalizedPageNumber = pageNumber == null || pageNumber < 1 ? 1 : pageNumber;
|
||||||
|
long normalizedPageSize = pageSize == null || pageSize < 1 ? 10 : pageSize;
|
||||||
|
QueryWrapper query = QueryWrapper.create()
|
||||||
|
.eq(DocumentChunk::getDocumentId, documentId)
|
||||||
|
.orderBy("sorting asc");
|
||||||
|
return Result.ok(documentChunkService.page(
|
||||||
|
new Page<>(normalizedPageNumber, normalizedPageSize),
|
||||||
|
query
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("update")
|
@PostMapping("update")
|
||||||
@@ -79,43 +84,23 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
|
|||||||
resource = CategoryResourceType.KNOWLEDGE,
|
resource = CategoryResourceType.KNOWLEDGE,
|
||||||
action = ResourceAction.MANAGE,
|
action = ResourceAction.MANAGE,
|
||||||
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
|
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
|
||||||
idExpr = "#documentChunk.id",
|
idExpr = "#request.id",
|
||||||
denyMessage = "无权限管理知识库"
|
denyMessage = "无权限管理知识库"
|
||||||
)
|
)
|
||||||
public Result<?> update(@JsonBody DocumentChunk documentChunk) {
|
public Result<?> update(
|
||||||
boolean success = service.updateById(documentChunk);
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
if (success){
|
DocumentChunkContentUpdateRequest request
|
||||||
DocumentChunk record = documentChunkService.getById(documentChunk.getId());
|
) {
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(record.getDocumentCollectionId());
|
DocumentChunk current = documentChunkService.getById(request.getId());
|
||||||
if (knowledge == null) {
|
if (current == null) {
|
||||||
return Result.fail(1, "知识库不存在");
|
return Result.fail(1, "记录不存在");
|
||||||
}
|
}
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
DocumentChunk updated = documentChunkService.updateContent(
|
||||||
if (documentStore == null) {
|
current.getDocumentCollectionId(),
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
current.getId(),
|
||||||
}
|
request.getContent()
|
||||||
try {
|
);
|
||||||
// 设置向量模型
|
return Result.ok(updated);
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
Document document = Document.of(documentChunk.getContent());
|
|
||||||
document.setId(documentChunk.getId());
|
|
||||||
Map<String, Object> metadata = new HashMap<>();
|
|
||||||
metadata.put("keywords", documentChunk.getMetadataKeyWords());
|
|
||||||
metadata.put("questions", documentChunk.getMetadataQuestions());
|
|
||||||
document.setMetadataMap(metadata);
|
|
||||||
StoreResult result = documentStore.update(document, options); // 更新已有记录
|
|
||||||
return Result.ok(result);
|
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Result.ok(false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("removeChunk")
|
@PostMapping("removeChunk")
|
||||||
@@ -127,36 +112,58 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
|
|||||||
idExpr = "#chunkId",
|
idExpr = "#chunkId",
|
||||||
denyMessage = "无权限管理知识库"
|
denyMessage = "无权限管理知识库"
|
||||||
)
|
)
|
||||||
public Result<?> remove(@JsonBody(value = "id", required = true) BigInteger chunkId) {
|
public Result<?> removeChunk(@JsonBody(value = "id", required = true) BigInteger chunkId) {
|
||||||
DocumentChunk docChunk = documentChunkService.getById(chunkId);
|
DocumentChunk docChunk = documentChunkService.getById(chunkId);
|
||||||
if (docChunk == null) {
|
if (docChunk == null) {
|
||||||
return Result.fail(1, "记录不存在");
|
return Result.fail(1, "记录不存在");
|
||||||
}
|
}
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(docChunk.getDocumentCollectionId());
|
return Result.ok(documentChunkService.deleteChunk(
|
||||||
if (knowledge == null) {
|
docChunk.getDocumentCollectionId(),
|
||||||
return Result.fail(2, "知识库不存在");
|
chunkId
|
||||||
|
));
|
||||||
}
|
}
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
|
||||||
if (documentStore == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量库");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
// 设置向量模型
|
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(4, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
List<BigInteger> deleteList = new ArrayList<>();
|
|
||||||
deleteList.add(chunkId);
|
|
||||||
documentStore.delete(deleteList, options);
|
|
||||||
documentChunkService.removeChunk(knowledge, chunkId);
|
|
||||||
|
|
||||||
return super.remove(chunkId);
|
@PostMapping("syncStatus")
|
||||||
} finally {
|
@SaCheckPermission("/api/v1/documentCollection/query")
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
@RequireResourceAccess(
|
||||||
}
|
resource = CategoryResourceType.KNOWLEDGE,
|
||||||
|
action = ResourceAction.READ,
|
||||||
|
lookup = ResourceLookup.DOCUMENT_ID,
|
||||||
|
idExpr = "#request.documentId",
|
||||||
|
denyMessage = "无权限访问知识库"
|
||||||
|
)
|
||||||
|
public Result<List<DocumentChunkSyncStatus>> syncStatus(
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncStatusRequest request
|
||||||
|
) {
|
||||||
|
return Result.ok(documentChunkService.listIndexSyncStatus(
|
||||||
|
null,
|
||||||
|
request.getDocumentId(),
|
||||||
|
request.getIds()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("retrySync")
|
||||||
|
@SaCheckPermission("/api/v1/documentCollection/save")
|
||||||
|
@RequireResourceAccess(
|
||||||
|
resource = CategoryResourceType.KNOWLEDGE,
|
||||||
|
action = ResourceAction.MANAGE,
|
||||||
|
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
|
||||||
|
idExpr = "#request.id",
|
||||||
|
denyMessage = "无权限管理知识库"
|
||||||
|
)
|
||||||
|
public Result<?> retrySync(
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncRetryRequest request
|
||||||
|
) {
|
||||||
|
DocumentChunk current = documentChunkService.getById(request.getId());
|
||||||
|
if (current == null || request.getIndexSyncVersion() == null) {
|
||||||
|
return Result.fail(1, "记录不存在或同步版本缺失");
|
||||||
|
}
|
||||||
|
return Result.ok(documentChunkService.retryIndexSync(
|
||||||
|
current.getDocumentCollectionId(),
|
||||||
|
current.getId(),
|
||||||
|
request.getIndexSyncVersion()
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -118,12 +118,10 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
|
|||||||
List<Serializable> ids = Collections.singletonList(id);
|
List<Serializable> ids = Collections.singletonList(id);
|
||||||
Result<?> result = onRemoveBefore(ids);
|
Result<?> result = onRemoveBefore(ids);
|
||||||
if (result != null) return result;
|
if (result != null) return result;
|
||||||
boolean isSuccess = documentService.removeDoc(id);
|
boolean success = documentService.removeDoc(id);
|
||||||
if (!isSuccess){
|
if (success) {
|
||||||
return Result.ok(false);
|
|
||||||
}
|
|
||||||
boolean success = service.removeById(id);
|
|
||||||
onRemoveAfter(ids);
|
onRemoveAfter(ids);
|
||||||
|
}
|
||||||
return Result.ok(success);
|
return Result.ok(success);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import cn.hutool.core.io.IoUtil;
|
import cn.hutool.core.io.IoUtil;
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreOptions;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryColumn;
|
import com.mybatisflex.core.query.QueryColumn;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
@@ -22,6 +18,11 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
|||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
|
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
|
||||||
import tech.easyflow.ai.dto.KnowledgeShareLimitedConfigRequest;
|
import tech.easyflow.ai.dto.KnowledgeShareLimitedConfigRequest;
|
||||||
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
|
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
|
||||||
import tech.easyflow.ai.entity.Document;
|
import tech.easyflow.ai.entity.Document;
|
||||||
@@ -43,7 +44,6 @@ import tech.easyflow.ai.service.KnowledgeEmbeddingService;
|
|||||||
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
||||||
import tech.easyflow.ai.service.KnowledgeShareService;
|
import tech.easyflow.ai.service.KnowledgeShareService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
|
||||||
import tech.easyflow.ai.vo.FaqImportResultVo;
|
import tech.easyflow.ai.vo.FaqImportResultVo;
|
||||||
import tech.easyflow.ai.vo.KnowledgeShareAuthContext;
|
import tech.easyflow.ai.vo.KnowledgeShareAuthContext;
|
||||||
import tech.easyflow.ai.vo.KnowledgeShareViewDetail;
|
import tech.easyflow.ai.vo.KnowledgeShareViewDetail;
|
||||||
@@ -62,7 +62,6 @@ import java.net.URLEncoder;
|
|||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
@@ -505,43 +504,27 @@ public class ShareKnowledgeController {
|
|||||||
@PostMapping("/documentChunk/update")
|
@PostMapping("/documentChunk/update")
|
||||||
public Result<?> updateDocumentChunk(
|
public Result<?> updateDocumentChunk(
|
||||||
@RequestParam String shareKey,
|
@RequestParam String shareKey,
|
||||||
@JsonBody DocumentChunk documentChunk
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkContentUpdateRequest request
|
||||||
) {
|
) {
|
||||||
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
||||||
shareKey,
|
shareKey,
|
||||||
null,
|
null,
|
||||||
KnowledgeShareActionScope.CONTENT_UPDATE.name()
|
KnowledgeShareActionScope.CONTENT_UPDATE.name()
|
||||||
);
|
);
|
||||||
DocumentChunk current = documentChunkService.getById(documentChunk.getId());
|
DocumentChunk current = documentChunkService.getById(request.getId());
|
||||||
if (current == null || current.getDocumentCollectionId() == null
|
if (current == null || current.getDocumentCollectionId() == null
|
||||||
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
||||||
throw new BusinessException("记录不存在");
|
throw new BusinessException("记录不存在");
|
||||||
}
|
}
|
||||||
boolean success = documentChunkService.updateById(documentChunk);
|
DocumentChunk updated = documentChunkService.updateContent(
|
||||||
if (success) {
|
context.getKnowledge().getId(),
|
||||||
DocumentStore documentStore = context.getKnowledge().toDocumentStore();
|
current.getId(),
|
||||||
if (documentStore == null) {
|
request.getContent()
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
);
|
||||||
}
|
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(context.getKnowledge().getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(context.getKnowledge().getVectorStoreCollection());
|
|
||||||
com.easyagents.core.document.Document doc = com.easyagents.core.document.Document.of(documentChunk.getContent());
|
|
||||||
doc.setId(documentChunk.getId());
|
|
||||||
StoreResult result = documentStore.update(doc, options);
|
|
||||||
audit(context, "更新分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
audit(context, "更新分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
||||||
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", documentChunk.getId()));
|
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", request.getId()));
|
||||||
return Result.ok(result);
|
return Result.ok(updated);
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Result.ok(false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -562,25 +545,50 @@ public class ShareKnowledgeController {
|
|||||||
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
||||||
return Result.fail(1, "记录不存在");
|
return Result.fail(1, "记录不存在");
|
||||||
}
|
}
|
||||||
DocumentStore documentStore = context.getKnowledge().toDocumentStore();
|
DocumentChunkDeleteResult removed = documentChunkService.deleteChunk(
|
||||||
if (documentStore == null) {
|
context.getKnowledge().getId(),
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
chunkId
|
||||||
}
|
);
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(context.getKnowledge().getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(context.getKnowledge().getVectorStoreCollection());
|
|
||||||
documentStore.delete(Collections.singletonList(chunkId), options);
|
|
||||||
documentChunkService.removeById(chunkId);
|
|
||||||
audit(context, "删除分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
audit(context, "删除分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
||||||
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", chunkId));
|
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", chunkId));
|
||||||
return Result.ok(true);
|
return Result.ok(removed);
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/syncStatus")
|
||||||
|
public Result<List<DocumentChunkSyncStatus>> documentChunkSyncStatus(
|
||||||
|
@RequestParam String shareKey,
|
||||||
|
@JsonBody DocumentChunkSyncStatusRequest request
|
||||||
|
) {
|
||||||
|
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
||||||
|
shareKey, null, KnowledgeShareActionScope.VIEW.name()
|
||||||
|
);
|
||||||
|
Document document = documentService.getById(request.getDocumentId());
|
||||||
|
if (document == null || document.getCollectionId() == null
|
||||||
|
|| document.getCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
||||||
|
throw new BusinessException("文档不存在");
|
||||||
|
}
|
||||||
|
return Result.ok(documentChunkService.listIndexSyncStatus(
|
||||||
|
context.getKnowledge().getId(), request.getDocumentId(), request.getIds()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/retrySync")
|
||||||
|
public Result<DocumentChunk> retryDocumentChunkSync(
|
||||||
|
@RequestParam String shareKey,
|
||||||
|
@JsonBody DocumentChunkSyncRetryRequest request
|
||||||
|
) {
|
||||||
|
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
||||||
|
shareKey, null, KnowledgeShareActionScope.CONTENT_UPDATE.name()
|
||||||
|
);
|
||||||
|
if (request.getIndexSyncVersion() == null) {
|
||||||
|
throw new BusinessException("同步版本不能为空");
|
||||||
|
}
|
||||||
|
DocumentChunk chunk = documentChunkService.retryIndexSync(
|
||||||
|
context.getKnowledge().getId(), request.getId(), request.getIndexSyncVersion()
|
||||||
|
);
|
||||||
|
audit(context, "重试分享文档 Chunk 索引同步", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
||||||
|
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", request.getId()));
|
||||||
|
return Result.ok(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
|
import org.testng.Assert;
|
||||||
|
import org.testng.annotations.Test;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
|
import tech.easyflow.common.web.controller.BaseController;
|
||||||
|
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||||
|
import tech.easyflow.system.permission.resource.RequireResourceAccess;
|
||||||
|
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.lang.reflect.Parameter;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文档分块维护接口契约测试。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkControllerContractTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void updateEndpointsShouldOnlyAcceptChunkIdAndContent() throws Exception {
|
||||||
|
Method adminUpdate = DocumentChunkController.class.getDeclaredMethod(
|
||||||
|
"update",
|
||||||
|
DocumentChunkContentUpdateRequest.class
|
||||||
|
);
|
||||||
|
Method shareUpdate = ShareKnowledgeController.class.getDeclaredMethod(
|
||||||
|
"updateDocumentChunk",
|
||||||
|
String.class,
|
||||||
|
DocumentChunkContentUpdateRequest.class
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertNotNull(adminUpdate);
|
||||||
|
Assert.assertNotNull(shareUpdate);
|
||||||
|
assertStrictJsonBody(adminUpdate.getParameters()[0]);
|
||||||
|
assertStrictJsonBody(shareUpdate.getParameters()[1]);
|
||||||
|
Assert.assertEquals(
|
||||||
|
adminUpdate.getAnnotation(RequireResourceAccess.class).idExpr(),
|
||||||
|
"#request.id"
|
||||||
|
);
|
||||||
|
Set<String> fields = Arrays.stream(
|
||||||
|
DocumentChunkContentUpdateRequest.class.getDeclaredFields()
|
||||||
|
)
|
||||||
|
.map(Field::getName)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
Assert.assertEquals(Set.of("id", "content"), fields);
|
||||||
|
Assert.assertEquals(
|
||||||
|
DocumentChunkController.class.getMethod(
|
||||||
|
"update",
|
||||||
|
DocumentChunkContentUpdateRequest.class
|
||||||
|
).getDeclaringClass(),
|
||||||
|
DocumentChunkController.class
|
||||||
|
);
|
||||||
|
Assert.assertEquals(
|
||||||
|
Arrays.stream(DocumentChunkController.class.getDeclaredMethods())
|
||||||
|
.filter(method -> method.getName().equals("update"))
|
||||||
|
.filter(method -> !method.isBridge() && !method.isSynthetic())
|
||||||
|
.count(),
|
||||||
|
1L
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void controllerShouldNotExposeGenericWriteEndpoints() {
|
||||||
|
Assert.assertEquals(DocumentChunkController.class.getSuperclass(), BaseController.class);
|
||||||
|
Set<String> postMappings = Arrays.stream(DocumentChunkController.class.getMethods())
|
||||||
|
.map(method -> method.getAnnotation(PostMapping.class))
|
||||||
|
.filter(annotation -> annotation != null)
|
||||||
|
.flatMap(annotation -> Arrays.stream(annotation.value()))
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
postMappings,
|
||||||
|
Set.of("update", "removeChunk", "syncStatus", "retrySync")
|
||||||
|
);
|
||||||
|
Assert.assertFalse(postMappings.contains("save"));
|
||||||
|
Assert.assertFalse(postMappings.contains("remove"));
|
||||||
|
Assert.assertFalse(postMappings.contains("removeBatch"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void adminUpdateAndDeleteShouldUseUnifiedMaintenanceService() {
|
||||||
|
DocumentChunkService service = mock(DocumentChunkService.class);
|
||||||
|
DocumentChunkController controller = new DocumentChunkController(service);
|
||||||
|
|
||||||
|
DocumentChunk current = new DocumentChunk();
|
||||||
|
current.setId(BigInteger.ONE);
|
||||||
|
current.setDocumentCollectionId(BigInteger.TWO);
|
||||||
|
when(service.getById(BigInteger.ONE)).thenReturn(current);
|
||||||
|
|
||||||
|
DocumentChunkContentUpdateRequest request = new DocumentChunkContentUpdateRequest();
|
||||||
|
request.setId(BigInteger.ONE);
|
||||||
|
request.setContent("updated");
|
||||||
|
controller.update(request);
|
||||||
|
controller.removeChunk(BigInteger.ONE);
|
||||||
|
|
||||||
|
verify(service).updateContent(BigInteger.TWO, BigInteger.ONE, "updated");
|
||||||
|
verify(service).deleteChunk(BigInteger.TWO, BigInteger.ONE);
|
||||||
|
verify(service, never()).updateById(any(DocumentChunk.class));
|
||||||
|
verify(service, never()).removeById(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertStrictJsonBody(Parameter parameter) {
|
||||||
|
JsonBody jsonBody = parameter.getAnnotation(JsonBody.class);
|
||||||
|
Assert.assertNotNull(jsonBody);
|
||||||
|
Assert.assertTrue(jsonBody.required());
|
||||||
|
Assert.assertFalse(jsonBody.skipConvertError());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,10 +1,6 @@
|
|||||||
package tech.easyflow.publicapi.controller;
|
package tech.easyflow.publicapi.controller;
|
||||||
|
|
||||||
import cn.hutool.core.io.IoUtil;
|
import cn.hutool.core.io.IoUtil;
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreOptions;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
@@ -18,11 +14,15 @@ import org.springframework.web.bind.annotation.RequestParam;
|
|||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkAsyncUpdateResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
|
||||||
import tech.easyflow.ai.entity.Document;
|
import tech.easyflow.ai.entity.Document;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.FaqItem;
|
import tech.easyflow.ai.entity.FaqItem;
|
||||||
import tech.easyflow.ai.entity.Model;
|
|
||||||
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
|
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
@@ -33,9 +33,7 @@ import tech.easyflow.ai.service.FaqCategoryService;
|
|||||||
import tech.easyflow.ai.service.FaqItemService;
|
import tech.easyflow.ai.service.FaqItemService;
|
||||||
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
||||||
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
|
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
|
||||||
import tech.easyflow.ai.service.impl.KnowledgeSharePermissionServiceImpl;
|
import tech.easyflow.ai.service.impl.KnowledgeSharePermissionServiceImpl;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
|
||||||
import tech.easyflow.ai.vo.FaqImportResultVo;
|
import tech.easyflow.ai.vo.FaqImportResultVo;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
@@ -81,8 +79,6 @@ public class PublicKnowledgeShareController {
|
|||||||
private FaqItemService faqItemService;
|
private FaqItemService faqItemService;
|
||||||
@Resource
|
@Resource
|
||||||
private FaqCategoryService faqCategoryService;
|
private FaqCategoryService faqCategoryService;
|
||||||
@Resource
|
|
||||||
private ModelService modelService;
|
|
||||||
@Resource(name = "default")
|
@Resource(name = "default")
|
||||||
private FileStorageService fileStorageService;
|
private FileStorageService fileStorageService;
|
||||||
|
|
||||||
@@ -351,37 +347,20 @@ public class PublicKnowledgeShareController {
|
|||||||
public Result<?> updateDocumentChunk(
|
public Result<?> updateDocumentChunk(
|
||||||
@RequestHeader("ApiKey") String apiKey,
|
@RequestHeader("ApiKey") String apiKey,
|
||||||
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
||||||
@JsonBody DocumentChunk documentChunk,
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunk documentChunk,
|
||||||
HttpServletRequest request
|
HttpServletRequest request
|
||||||
) {
|
) {
|
||||||
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
||||||
requireDocumentKnowledge(knowledgeId);
|
requireDocumentKnowledge(knowledgeId);
|
||||||
DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId);
|
DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId);
|
||||||
boolean success = documentChunkService.updateById(documentChunk);
|
DocumentChunk updated = documentChunkService.updateContent(
|
||||||
if (success) {
|
knowledgeId,
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
|
current.getId(),
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
documentChunk.getContent()
|
||||||
if (documentStore == null) {
|
);
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
com.easyagents.core.document.Document doc = com.easyagents.core.document.Document.of(documentChunk.getContent());
|
|
||||||
doc.setId(current.getId());
|
|
||||||
StoreResult result = documentStore.update(doc, options);
|
|
||||||
audit(apiKey, "API更新文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", documentChunk.getId()));
|
audit(apiKey, "API更新文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", documentChunk.getId()));
|
||||||
return Result.ok(result);
|
return Result.ok(DocumentChunkAsyncUpdateResult.from(updated));
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Result.ok(false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -397,25 +376,51 @@ public class PublicKnowledgeShareController {
|
|||||||
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
||||||
requireDocumentKnowledge(knowledgeId);
|
requireDocumentKnowledge(knowledgeId);
|
||||||
requireDocumentChunk(chunkId, knowledgeId);
|
requireDocumentChunk(chunkId, knowledgeId);
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
|
DocumentChunkDeleteResult removed = documentChunkService.deleteChunk(
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
knowledgeId,
|
||||||
if (documentStore == null) {
|
chunkId
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
);
|
||||||
}
|
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
documentStore.delete(Collections.singletonList(chunkId), options);
|
|
||||||
documentChunkService.removeById(chunkId);
|
|
||||||
audit(apiKey, "API删除文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", chunkId));
|
audit(apiKey, "API删除文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", chunkId));
|
||||||
return Result.ok(true);
|
return Result.ok(removed != null);
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/syncStatus")
|
||||||
|
public Result<List<DocumentChunkSyncStatus>> documentChunkSyncStatus(
|
||||||
|
@RequestHeader("ApiKey") String apiKey,
|
||||||
|
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncStatusRequest statusRequest,
|
||||||
|
HttpServletRequest request
|
||||||
|
) {
|
||||||
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId,
|
||||||
|
KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
|
||||||
|
requireDocumentKnowledge(knowledgeId);
|
||||||
|
requireDocument(statusRequest.getDocumentId(), knowledgeId);
|
||||||
|
return Result.ok(documentChunkService.listIndexSyncStatus(
|
||||||
|
knowledgeId, statusRequest.getDocumentId(), statusRequest.getIds()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/retrySync")
|
||||||
|
public Result<DocumentChunk> retryDocumentChunkSync(
|
||||||
|
@RequestHeader("ApiKey") String apiKey,
|
||||||
|
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncRetryRequest retryRequest,
|
||||||
|
HttpServletRequest request
|
||||||
|
) {
|
||||||
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId,
|
||||||
|
KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
||||||
|
requireDocumentKnowledge(knowledgeId);
|
||||||
|
if (retryRequest.getIndexSyncVersion() == null) {
|
||||||
|
throw new BusinessException("同步版本不能为空");
|
||||||
|
}
|
||||||
|
DocumentChunk chunk = documentChunkService.retryIndexSync(
|
||||||
|
knowledgeId, retryRequest.getId(), retryRequest.getIndexSyncVersion()
|
||||||
|
);
|
||||||
|
audit(apiKey, "API重试文档 Chunk 索引同步", "KNOWLEDGE_API_SHARE_WRITE",
|
||||||
|
request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", retryRequest.getId()));
|
||||||
|
return Result.ok(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -66,6 +66,8 @@ public class PublicKnowledgeShareControllerContractTest {
|
|||||||
JsonBody chunkBody = chunk.getAnnotation(JsonBody.class);
|
JsonBody chunkBody = chunk.getAnnotation(JsonBody.class);
|
||||||
Assert.assertNotNull(chunkBody);
|
Assert.assertNotNull(chunkBody);
|
||||||
Assert.assertEquals("", chunkBody.value());
|
Assert.assertEquals("", chunkBody.value());
|
||||||
|
Assert.assertTrue(chunkBody.required());
|
||||||
|
Assert.assertFalse(chunkBody.skipConvertError());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -53,6 +53,10 @@
|
|||||||
<groupId>com.easyagents</groupId>
|
<groupId>com.easyagents</groupId>
|
||||||
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.milvus</groupId>
|
||||||
|
<artifactId>milvus-sdk-java</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.google.re2j</groupId>
|
<groupId>com.google.re2j</groupId>
|
||||||
<artifactId>re2j</artifactId>
|
<artifactId>re2j</artifactId>
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
|
import com.easyagents.store.milvus.MilvusClientManager;
|
||||||
|
import jakarta.annotation.PreDestroy;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.context.annotation.Lazy;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 应用级 Milvus 客户端池。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
@Lazy
|
||||||
|
public class AiMilvusClientManager extends MilvusClientManager {
|
||||||
|
|
||||||
|
public AiMilvusClientManager(AiMilvusConfig config) {
|
||||||
|
super(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
public void destroy() {
|
||||||
|
close();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,13 @@ public class AiMilvusConfig extends MilvusVectorStoreConfig {
|
|||||||
config.setPassword(getPassword());
|
config.setPassword(getPassword());
|
||||||
config.setAutoCreateCollection(isAutoCreateCollection());
|
config.setAutoCreateCollection(isAutoCreateCollection());
|
||||||
config.setDefaultCollectionName(collectionName);
|
config.setDefaultCollectionName(collectionName);
|
||||||
|
config.setPoolMaxTotal(getPoolMaxTotal());
|
||||||
|
config.setPoolMaxTotalPerKey(getPoolMaxTotalPerKey());
|
||||||
|
config.setPoolMaxIdlePerKey(getPoolMaxIdlePerKey());
|
||||||
|
config.setPoolMinIdlePerKey(getPoolMinIdlePerKey());
|
||||||
|
config.setPoolMaxWaitMillis(getPoolMaxWaitMillis());
|
||||||
|
config.setPoolEvictionIntervalMillis(getPoolEvictionIntervalMillis());
|
||||||
|
config.setPoolMinEvictableIdleMillis(getPoolMinEvictableIdleMillis());
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,16 +25,23 @@ public class RagHealthIndicator {
|
|||||||
public static class RagMilvusHealthIndicator extends CachedHealthIndicatorSupport implements HealthIndicator {
|
public static class RagMilvusHealthIndicator extends CachedHealthIndicatorSupport implements HealthIndicator {
|
||||||
|
|
||||||
private final AiMilvusConfig aiMilvusConfig;
|
private final AiMilvusConfig aiMilvusConfig;
|
||||||
|
private final AiMilvusClientManager milvusClientManager;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建 Milvus 健康检查器。
|
* 创建 Milvus 健康检查器。
|
||||||
*
|
*
|
||||||
* @param aiMilvusConfig Milvus 配置
|
* @param aiMilvusConfig Milvus 配置
|
||||||
|
* @param milvusClientManager 应用级 Milvus 客户端池
|
||||||
* @param healthProperties RAG 健康检查配置
|
* @param healthProperties RAG 健康检查配置
|
||||||
*/
|
*/
|
||||||
public RagMilvusHealthIndicator(AiMilvusConfig aiMilvusConfig, RagHealthProperties healthProperties) {
|
public RagMilvusHealthIndicator(
|
||||||
|
AiMilvusConfig aiMilvusConfig,
|
||||||
|
AiMilvusClientManager milvusClientManager,
|
||||||
|
RagHealthProperties healthProperties
|
||||||
|
) {
|
||||||
super(healthProperties);
|
super(healthProperties);
|
||||||
this.aiMilvusConfig = aiMilvusConfig;
|
this.aiMilvusConfig = aiMilvusConfig;
|
||||||
|
this.milvusClientManager = milvusClientManager;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -51,8 +58,10 @@ public class RagHealthIndicator {
|
|||||||
protected Health doHealthCheck() {
|
protected Health doHealthCheck() {
|
||||||
MilvusVectorStore vectorStore = null;
|
MilvusVectorStore vectorStore = null;
|
||||||
try {
|
try {
|
||||||
|
milvusClientManager.reconfigureIfNeeded(aiMilvusConfig);
|
||||||
vectorStore = new MilvusVectorStore(
|
vectorStore = new MilvusVectorStore(
|
||||||
aiMilvusConfig.copyForCollection("__rag_health_probe__")
|
aiMilvusConfig.copyForCollection("__rag_health_probe__"),
|
||||||
|
milvusClientManager
|
||||||
);
|
);
|
||||||
if (vectorStore.checkAvailable()) {
|
if (vectorStore.checkAvailable()) {
|
||||||
return Health.up().withDetail("uri", aiMilvusConfig.getUri()).build();
|
return Health.up().withDetail("uri", aiMilvusConfig.getUri()).build();
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块检索索引同步状态常量。
|
||||||
|
*/
|
||||||
|
public final class DocumentChunkSyncState {
|
||||||
|
|
||||||
|
private static final String PARENT_LOCK_PREFIX =
|
||||||
|
"easyflow:lock:document-chunk-parent:";
|
||||||
|
private static final String SYNC_LOCK_PREFIX =
|
||||||
|
"easyflow:lock:document-chunk-sync:";
|
||||||
|
|
||||||
|
public static final String SYNCED = "SYNCED";
|
||||||
|
public static final String PENDING = "PENDING";
|
||||||
|
public static final String FAILED = "FAILED";
|
||||||
|
|
||||||
|
public static final String TASK_RUNNING = "RUNNING";
|
||||||
|
public static final String TASK_SUCCEEDED = "SUCCEEDED";
|
||||||
|
public static final String TASK_SUPERSEDED = "SUPERSEDED";
|
||||||
|
|
||||||
|
public static final String OPERATION_UPSERT = "UPSERT";
|
||||||
|
public static final String OPERATION_DELETE = "DELETE";
|
||||||
|
|
||||||
|
public static String parentLockKey(Object documentId) {
|
||||||
|
return PARENT_LOCK_PREFIX + documentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String syncLockKey(Object chunkId) {
|
||||||
|
return SYNC_LOCK_PREFIX + chunkId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DocumentChunkSyncState() {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,414 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||||
|
import com.easyagents.core.model.embedding.EmbeddingOptions;
|
||||||
|
import com.easyagents.core.store.DocumentStore;
|
||||||
|
import com.easyagents.core.store.StoreOptions;
|
||||||
|
import com.easyagents.core.store.StoreResult;
|
||||||
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
|
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStore;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusClientManager;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.entity.Model;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.ai.service.ModelService;
|
||||||
|
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 持久化分块索引同步任务的投递、执行和恢复。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class DocumentChunkSyncTaskAppService {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentChunkSyncTaskAppService.class);
|
||||||
|
private static final int MAX_ATTEMPTS = 5;
|
||||||
|
private static final int DISPATCH_LIMIT = 100;
|
||||||
|
private static final long REDISPATCH_MILLIS = 5_000L;
|
||||||
|
private static final long LEASE_MILLIS = 300_000L;
|
||||||
|
|
||||||
|
private final DocumentChunkSyncTaskMapper taskMapper;
|
||||||
|
private final DocumentChunkMapper chunkMapper;
|
||||||
|
private final DocumentCollectionService collectionService;
|
||||||
|
private final ModelService modelService;
|
||||||
|
private final SearcherFactory searcherFactory;
|
||||||
|
private final DocumentChunkSyncTaskProducer producer;
|
||||||
|
private final PlatformTransactionManager transactionManager;
|
||||||
|
private final RedisLockExecutor redisLockExecutor;
|
||||||
|
private final AiMilvusConfig milvusConfig;
|
||||||
|
private final ObjectProvider<AiMilvusClientManager> milvusClientManagerProvider;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskAppService(
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper,
|
||||||
|
DocumentChunkMapper chunkMapper,
|
||||||
|
DocumentCollectionService collectionService,
|
||||||
|
ModelService modelService,
|
||||||
|
SearcherFactory searcherFactory,
|
||||||
|
DocumentChunkSyncTaskProducer producer,
|
||||||
|
PlatformTransactionManager transactionManager,
|
||||||
|
RedisLockExecutor redisLockExecutor,
|
||||||
|
AiMilvusConfig milvusConfig,
|
||||||
|
ObjectProvider<AiMilvusClientManager> milvusClientManagerProvider
|
||||||
|
) {
|
||||||
|
this.taskMapper = taskMapper;
|
||||||
|
this.chunkMapper = chunkMapper;
|
||||||
|
this.collectionService = collectionService;
|
||||||
|
this.modelService = modelService;
|
||||||
|
this.searcherFactory = searcherFactory;
|
||||||
|
this.producer = producer;
|
||||||
|
this.transactionManager = transactionManager;
|
||||||
|
this.redisLockExecutor = redisLockExecutor;
|
||||||
|
this.milvusConfig = milvusConfig;
|
||||||
|
this.milvusClientManagerProvider = milvusClientManagerProvider;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DocumentChunkSyncTask createTask(
|
||||||
|
DocumentChunk chunk,
|
||||||
|
DocumentCollection collection,
|
||||||
|
String operation,
|
||||||
|
long version,
|
||||||
|
Date now
|
||||||
|
) {
|
||||||
|
taskMapper.supersedeOlder(chunk.getId(), version, now);
|
||||||
|
DocumentChunkSyncTask task = new DocumentChunkSyncTask();
|
||||||
|
task.setChunkId(chunk.getId());
|
||||||
|
task.setDocumentId(chunk.getDocumentId());
|
||||||
|
task.setDocumentCollectionId(chunk.getDocumentCollectionId());
|
||||||
|
task.setVectorCollection(collection.getVectorStoreCollection());
|
||||||
|
task.setOperation(operation);
|
||||||
|
task.setSyncVersion(version);
|
||||||
|
task.setStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
task.setAttemptCount(0);
|
||||||
|
task.setNextRetryAt(now);
|
||||||
|
task.setCreated(now);
|
||||||
|
task.setModified(now);
|
||||||
|
if (taskMapper.insert(task) <= 0) {
|
||||||
|
throw new IllegalStateException("创建分块索引同步任务失败");
|
||||||
|
}
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void dispatchBestEffort(BigInteger taskId) {
|
||||||
|
try {
|
||||||
|
producer.send(taskId);
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.warn("分块索引同步消息投递失败,等待数据库补投: taskId={}", taskId, exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void dispatchPendingTasks() {
|
||||||
|
Date now = new Date();
|
||||||
|
taskMapper.recoverExpired(now);
|
||||||
|
Date redispatchBefore = new Date(now.getTime() - REDISPATCH_MILLIS);
|
||||||
|
for (DocumentChunkSyncTask task : taskMapper.selectPendingDue(
|
||||||
|
now,
|
||||||
|
redispatchBefore,
|
||||||
|
DISPATCH_LIMIT
|
||||||
|
)) {
|
||||||
|
if (taskMapper.markDispatched(task.getId(), now, redispatchBefore) <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
producer.send(task.getId());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.warn("补投分块索引同步消息失败: taskId={}", task.getId(), exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleTask(BigInteger taskId) {
|
||||||
|
DocumentChunkSyncTask snapshot = taskMapper.selectOneById(taskId);
|
||||||
|
if (snapshot == null || !DocumentChunkSyncState.PENDING.equals(snapshot.getStatus())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
redisLockExecutor.executeWithRenewingLock(
|
||||||
|
DocumentChunkSyncState.syncLockKey(snapshot.getChunkId()),
|
||||||
|
Duration.ofSeconds(3),
|
||||||
|
Duration.ofMinutes(5),
|
||||||
|
() -> {
|
||||||
|
claimAndExecute(taskId, snapshot.getChunkId());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void claimAndExecute(BigInteger taskId, BigInteger chunkId) {
|
||||||
|
Date now = new Date();
|
||||||
|
String token = UUID.randomUUID().toString();
|
||||||
|
if (taskMapper.claim(
|
||||||
|
taskId,
|
||||||
|
token,
|
||||||
|
new Date(now.getTime() + LEASE_MILLIS),
|
||||||
|
now
|
||||||
|
) <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status ->
|
||||||
|
executeOwnedTask(taskId, chunkId, token)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void executeOwnedTask(
|
||||||
|
BigInteger taskId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
String token
|
||||||
|
) {
|
||||||
|
// FOR UPDATE 必须是本事务的第一次读取。MySQL REPEATABLE READ 下,
|
||||||
|
// 若先做普通查询会建立旧快照,使删除或新版本建单已经提交后仍读取旧分块。
|
||||||
|
taskMapper.lockChunkTasks(chunkId);
|
||||||
|
DocumentChunkSyncTask task = taskMapper.selectOneById(taskId);
|
||||||
|
if (task == null
|
||||||
|
|| !DocumentChunkSyncState.TASK_RUNNING.equals(task.getStatus())
|
||||||
|
|| !token.equals(task.getExecutionToken())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isSuperseded(task)) {
|
||||||
|
finishTask(task, token, DocumentChunkSyncState.TASK_SUPERSEDED, null, null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
synchronizeIndexes(task);
|
||||||
|
finishSuccess(task, token);
|
||||||
|
} catch (IndexSyncException exception) {
|
||||||
|
LOG.warn("分块索引同步失败: taskId={}, chunkId={}, code={}",
|
||||||
|
task.getId(), task.getChunkId(), exception.code, exception.getCause());
|
||||||
|
finishFailure(task, token, exception.code, exception.getMessage());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.error("分块索引同步发生未分类异常: taskId={}, chunkId={}",
|
||||||
|
task.getId(), task.getChunkId(), exception);
|
||||||
|
finishFailure(task, token, "INDEX_SYNC_FAILED", "检索索引同步失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isSuperseded(DocumentChunkSyncTask task) {
|
||||||
|
DocumentChunk chunk = chunkMapper.selectOneById(task.getChunkId());
|
||||||
|
if (DocumentChunkSyncState.OPERATION_DELETE.equals(task.getOperation())) {
|
||||||
|
return chunk != null;
|
||||||
|
}
|
||||||
|
return chunk == null
|
||||||
|
|| chunk.getIndexSyncVersion() == null
|
||||||
|
|| chunk.getIndexSyncVersion().longValue() != task.getSyncVersion().longValue()
|
||||||
|
|| !DocumentChunkSyncState.PENDING.equals(chunk.getIndexSyncStatus());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void synchronizeIndexes(DocumentChunkSyncTask task) {
|
||||||
|
if (DocumentChunkSyncState.OPERATION_DELETE.equals(task.getOperation())) {
|
||||||
|
synchronizeDelete(task);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
synchronizeUpsert(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void synchronizeUpsert(DocumentChunkSyncTask task) {
|
||||||
|
DocumentChunk chunk = chunkMapper.selectOneById(task.getChunkId());
|
||||||
|
DocumentCollection collection = collectionService.getById(task.getDocumentCollectionId());
|
||||||
|
if (chunk == null || collection == null) {
|
||||||
|
throw new IndexSyncException("INDEX_SOURCE_MISSING", "分块或知识库不存在", null);
|
||||||
|
}
|
||||||
|
StoreContext context = prepareUpsertContext(collection, task.getVectorCollection());
|
||||||
|
try {
|
||||||
|
com.easyagents.core.document.Document document = toSearchDocument(chunk, task.getDocumentCollectionId());
|
||||||
|
StoreResult vectorResult = context.documentStore.update(
|
||||||
|
Collections.singletonList(document),
|
||||||
|
context.storeOptions
|
||||||
|
);
|
||||||
|
if (vectorResult == null || !vectorResult.isSuccess()) {
|
||||||
|
throw new IndexSyncException("VECTOR_UPSERT_FAILED", "向量索引更新失败", null);
|
||||||
|
}
|
||||||
|
if (context.searcher != null
|
||||||
|
&& !context.searcher.addDocuments(Collections.singletonList(document))) {
|
||||||
|
throw new IndexSyncException("KEYWORD_UPSERT_FAILED", "关键词索引更新失败", null);
|
||||||
|
}
|
||||||
|
} catch (IndexSyncException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new IndexSyncException("INDEX_UPSERT_FAILED", "检索索引更新失败", exception);
|
||||||
|
} finally {
|
||||||
|
DocumentStoreLifecycleSupport.closeQuietly(context.documentStore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void synchronizeDelete(DocumentChunkSyncTask task) {
|
||||||
|
MilvusVectorStoreConfig storeConfig = milvusConfig.copyForCollection(task.getVectorCollection());
|
||||||
|
AiMilvusClientManager clientManager = milvusClientManagerProvider.getObject();
|
||||||
|
clientManager.reconfigureIfNeeded(milvusConfig);
|
||||||
|
DocumentStore documentStore = new MilvusVectorStore(
|
||||||
|
storeConfig,
|
||||||
|
clientManager
|
||||||
|
);
|
||||||
|
StoreOptions storeOptions = StoreOptions.ofCollectionName(task.getVectorCollection());
|
||||||
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
|
try {
|
||||||
|
StoreResult result = documentStore.delete(
|
||||||
|
Collections.singletonList(task.getChunkId().toString()),
|
||||||
|
storeOptions
|
||||||
|
);
|
||||||
|
if (result == null || !result.isSuccess()) {
|
||||||
|
throw new IndexSyncException("VECTOR_DELETE_FAILED", "向量索引删除失败", null);
|
||||||
|
}
|
||||||
|
if (searcher != null && !searcher.deleteDocument(task.getChunkId())) {
|
||||||
|
throw new IndexSyncException("KEYWORD_DELETE_FAILED", "关键词索引删除失败", null);
|
||||||
|
}
|
||||||
|
} catch (IndexSyncException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new IndexSyncException("INDEX_DELETE_FAILED", "检索索引删除失败", exception);
|
||||||
|
} finally {
|
||||||
|
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private StoreContext prepareUpsertContext(
|
||||||
|
DocumentCollection collection,
|
||||||
|
String vectorCollection
|
||||||
|
) {
|
||||||
|
DocumentStore documentStore = collection.toDocumentStore();
|
||||||
|
if (documentStore == null) {
|
||||||
|
throw new IndexSyncException("VECTOR_STORE_MISSING", "知识库没有配置向量库", null);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Model model = modelService.getModelInstance(collection.getVectorEmbedModelId());
|
||||||
|
if (model == null) {
|
||||||
|
throw new IndexSyncException("EMBEDDING_MODEL_MISSING", "知识库没有配置向量模型", null);
|
||||||
|
}
|
||||||
|
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
||||||
|
documentStore.setEmbeddingModel(embeddingModel);
|
||||||
|
StoreOptions options = StoreOptions.ofCollectionName(vectorCollection);
|
||||||
|
EmbeddingOptions embeddingOptions = new EmbeddingOptions();
|
||||||
|
embeddingOptions.setModel(model.getModelName());
|
||||||
|
embeddingOptions.setDimensions(collection.getDimensionOfVectorModel());
|
||||||
|
options.setEmbeddingOptions(embeddingOptions);
|
||||||
|
options.setIndexName(vectorCollection);
|
||||||
|
return new StoreContext(documentStore, options, searcherFactory.getSearcher());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
||||||
|
throw exception;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private com.easyagents.core.document.Document toSearchDocument(
|
||||||
|
DocumentChunk chunk,
|
||||||
|
BigInteger knowledgeId
|
||||||
|
) {
|
||||||
|
com.easyagents.core.document.Document document =
|
||||||
|
com.easyagents.core.document.Document.of(chunk.getContent());
|
||||||
|
document.setId(chunk.getId());
|
||||||
|
document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, knowledgeId.toString());
|
||||||
|
return document;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishSuccess(DocumentChunkSyncTask task, String token) {
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status -> {
|
||||||
|
if (taskMapper.finishOwned(
|
||||||
|
task.getId(), token, DocumentChunkSyncState.TASK_SUCCEEDED, null, null, new Date()
|
||||||
|
) > 0 && DocumentChunkSyncState.OPERATION_UPSERT.equals(task.getOperation())) {
|
||||||
|
chunkMapper.updateSyncState(
|
||||||
|
task.getChunkId(), task.getSyncVersion(), DocumentChunkSyncState.SYNCED, null, null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishTask(
|
||||||
|
DocumentChunkSyncTask task,
|
||||||
|
String token,
|
||||||
|
String status,
|
||||||
|
String errorCode,
|
||||||
|
String errorMessage
|
||||||
|
) {
|
||||||
|
taskMapper.finishOwned(task.getId(), token, status, errorCode, errorMessage, new Date());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishFailure(
|
||||||
|
DocumentChunkSyncTask task,
|
||||||
|
String token,
|
||||||
|
String errorCode,
|
||||||
|
String errorMessage
|
||||||
|
) {
|
||||||
|
int attempts = task.getAttemptCount() == null ? 1 : task.getAttemptCount();
|
||||||
|
boolean deleteOperation = DocumentChunkSyncState.OPERATION_DELETE.equals(
|
||||||
|
task.getOperation()
|
||||||
|
);
|
||||||
|
// 删除后已没有页面实体承载手动重试入口,因此清理任务必须保留为持久化 tombstone。
|
||||||
|
boolean exhausted = !deleteOperation && attempts >= MAX_ATTEMPTS;
|
||||||
|
String nextStatus = exhausted ? DocumentChunkSyncState.FAILED : DocumentChunkSyncState.PENDING;
|
||||||
|
long delaySeconds = Math.min(1L << Math.min(attempts, 6), 60L);
|
||||||
|
Date now = new Date();
|
||||||
|
Date nextRetryAt = exhausted ? now : new Date(now.getTime() + delaySeconds * 1_000L);
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status -> {
|
||||||
|
if (taskMapper.failOrRetryOwned(
|
||||||
|
task.getId(), token, nextStatus, nextRetryAt, errorCode, errorMessage, now
|
||||||
|
) > 0 && exhausted && DocumentChunkSyncState.OPERATION_UPSERT.equals(task.getOperation())) {
|
||||||
|
chunkMapper.updateSyncState(
|
||||||
|
task.getChunkId(), task.getSyncVersion(), DocumentChunkSyncState.FAILED,
|
||||||
|
errorCode, errorMessage
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public DocumentChunk retry(BigInteger chunkId, long syncVersion) {
|
||||||
|
Date now = new Date();
|
||||||
|
DocumentChunk chunk = chunkMapper.selectOneById(chunkId);
|
||||||
|
if (chunk == null || chunk.getIndexSyncVersion() == null
|
||||||
|
|| chunk.getIndexSyncVersion().longValue() != syncVersion
|
||||||
|
|| !DocumentChunkSyncState.FAILED.equals(chunk.getIndexSyncStatus())) {
|
||||||
|
throw new IllegalStateException("分块同步状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
|
DocumentChunkSyncTask task = taskMapper.selectFailed(chunkId, syncVersion);
|
||||||
|
if (task == null) {
|
||||||
|
throw new IllegalStateException("未找到可重试的索引同步任务");
|
||||||
|
}
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status -> {
|
||||||
|
if (taskMapper.retryFailed(task.getId(), chunkId, syncVersion, now) <= 0) {
|
||||||
|
throw new IllegalStateException("索引同步任务状态已变化");
|
||||||
|
}
|
||||||
|
if (chunkMapper.updateSyncState(
|
||||||
|
chunkId, syncVersion, DocumentChunkSyncState.PENDING, null, null
|
||||||
|
) <= 0) {
|
||||||
|
throw new IllegalStateException("分块同步状态已变化");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
dispatchBestEffort(task.getId());
|
||||||
|
chunk.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
chunk.setIndexSyncErrorCode(null);
|
||||||
|
chunk.setIndexSyncErrorMessage(null);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
private record StoreContext(
|
||||||
|
DocumentStore documentStore,
|
||||||
|
StoreOptions storeOptions,
|
||||||
|
DocumentSearcher searcher
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class IndexSyncException extends RuntimeException {
|
||||||
|
private final String code;
|
||||||
|
|
||||||
|
private IndexSyncException(String code, String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
this.code = code;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.common.mq.config.MQProperties;
|
||||||
|
import tech.easyflow.common.mq.core.MQConsumerHandler;
|
||||||
|
import tech.easyflow.common.mq.core.MQDeferException;
|
||||||
|
import tech.easyflow.common.mq.core.MQMessage;
|
||||||
|
import tech.easyflow.common.mq.core.MQSubscription;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步消息消费者。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class DocumentChunkSyncTaskConsumer implements MQConsumerHandler {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentChunkSyncTaskConsumer.class);
|
||||||
|
|
||||||
|
private final DocumentChunkSyncTaskAppService appService;
|
||||||
|
private final MQProperties mqProperties;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskConsumer(
|
||||||
|
DocumentChunkSyncTaskAppService appService,
|
||||||
|
MQProperties mqProperties
|
||||||
|
) {
|
||||||
|
this.appService = appService;
|
||||||
|
this.mqProperties = mqProperties;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MQSubscription subscription() {
|
||||||
|
MQSubscription subscription = new MQSubscription();
|
||||||
|
subscription.setTopic(DocumentChunkSyncTaskProducer.TOPIC);
|
||||||
|
subscription.setConsumerGroup(DocumentChunkSyncTaskProducer.GROUP);
|
||||||
|
subscription.setShardCount(Math.max(
|
||||||
|
mqProperties.getRedis().getChatPersistShardCount(),
|
||||||
|
1
|
||||||
|
));
|
||||||
|
subscription.setBatchEnabled(false);
|
||||||
|
return subscription;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handle(List<MQMessage> messages) {
|
||||||
|
for (MQMessage message : messages) {
|
||||||
|
DocumentChunkSyncTaskMessage event = JSON.parseObject(
|
||||||
|
message.getBody(),
|
||||||
|
DocumentChunkSyncTaskMessage.class
|
||||||
|
);
|
||||||
|
if (event == null || event.getTaskId() == null) {
|
||||||
|
LOG.warn("忽略非法分块索引同步消息: messageId={}", message.getMessageId());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
appService.handleTask(event.getTaskId());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.error("分块索引同步任务状态处理失败: taskId={}", event.getTaskId(), exception);
|
||||||
|
throw new MQDeferException("分块索引同步任务暂时无法处理", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步消息。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncTaskMessage implements Serializable {
|
||||||
|
private BigInteger taskId;
|
||||||
|
private Date occurredAt;
|
||||||
|
|
||||||
|
public BigInteger getTaskId() { return taskId; }
|
||||||
|
public void setTaskId(BigInteger taskId) { this.taskId = taskId; }
|
||||||
|
public Date getOccurredAt() { return occurredAt; }
|
||||||
|
public void setOccurredAt(Date occurredAt) { this.occurredAt = occurredAt; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 补投待同步任务并回收过期租约。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class DocumentChunkSyncTaskMonitor {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentChunkSyncTaskMonitor.class);
|
||||||
|
|
||||||
|
private final DocumentChunkSyncTaskAppService appService;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskMonitor(DocumentChunkSyncTaskAppService appService) {
|
||||||
|
this.appService = appService;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Scheduled(fixedDelayString = "${easyflow.ai.document-chunk-sync.dispatch-interval:2s}",
|
||||||
|
initialDelayString = "${easyflow.ai.document-chunk-sync.dispatch-interval:2s}")
|
||||||
|
@DistributedScheduledLock(key = "easyflow:schedule:document-chunk-index-sync", leaseSeconds = 2L)
|
||||||
|
public void dispatchPendingTasks() {
|
||||||
|
try {
|
||||||
|
appService.dispatchPendingTasks();
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.error("分块索引同步补投失败", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import tech.easyflow.common.mq.core.MQMessage;
|
||||||
|
import tech.easyflow.common.mq.core.MQProducer;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步消息生产者。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class DocumentChunkSyncTaskProducer {
|
||||||
|
|
||||||
|
static final String TOPIC = "document-chunk-index-sync";
|
||||||
|
static final String GROUP = "document-chunk-index-sync-group";
|
||||||
|
|
||||||
|
private final MQProducer mqProducer;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskProducer(MQProducer mqProducer) {
|
||||||
|
this.mqProducer = mqProducer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void send(BigInteger taskId) {
|
||||||
|
Date now = new Date();
|
||||||
|
DocumentChunkSyncTaskMessage event = new DocumentChunkSyncTaskMessage();
|
||||||
|
event.setTaskId(taskId);
|
||||||
|
event.setOccurredAt(now);
|
||||||
|
MQMessage message = new MQMessage();
|
||||||
|
message.setMessageId("chunk-sync-" + taskId);
|
||||||
|
message.setTopic(TOPIC);
|
||||||
|
message.setKey(String.valueOf(taskId));
|
||||||
|
message.setCreatedAt(now);
|
||||||
|
message.setBody(JSON.toJSONString(event));
|
||||||
|
mqProducer.send(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 兼容 success 字段的异步分块更新结果。
|
||||||
|
*/
|
||||||
|
public record DocumentChunkAsyncUpdateResult(
|
||||||
|
boolean success,
|
||||||
|
BigInteger id,
|
||||||
|
String indexSyncStatus,
|
||||||
|
Long indexSyncVersion
|
||||||
|
) {
|
||||||
|
public static DocumentChunkAsyncUpdateResult from(DocumentChunk chunk) {
|
||||||
|
return new DocumentChunkAsyncUpdateResult(
|
||||||
|
true,
|
||||||
|
chunk.getId(),
|
||||||
|
chunk.getIndexSyncStatus(),
|
||||||
|
chunk.getIndexSyncVersion()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块正文更新请求,只允许修改 Markdown 内容。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkContentUpdateRequest implements Serializable {
|
||||||
|
|
||||||
|
private BigInteger id;
|
||||||
|
private String content;
|
||||||
|
|
||||||
|
public BigInteger getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(BigInteger id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getContent() {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setContent(String content) {
|
||||||
|
this.content = content;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块删除结果。
|
||||||
|
*/
|
||||||
|
public record DocumentChunkDeleteResult(
|
||||||
|
BigInteger id,
|
||||||
|
BigInteger documentId,
|
||||||
|
long remainingChunkCount
|
||||||
|
) implements Serializable {
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 重试分块索引同步请求。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncRetryRequest {
|
||||||
|
private BigInteger id;
|
||||||
|
private Long indexSyncVersion;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public Long getIndexSyncVersion() { return indexSyncVersion; }
|
||||||
|
public void setIndexSyncVersion(Long indexSyncVersion) { this.indexSyncVersion = indexSyncVersion; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步状态。
|
||||||
|
*/
|
||||||
|
public record DocumentChunkSyncStatus(
|
||||||
|
BigInteger id,
|
||||||
|
String indexSyncStatus,
|
||||||
|
Long indexSyncVersion,
|
||||||
|
String indexSyncErrorCode,
|
||||||
|
String indexSyncErrorMessage
|
||||||
|
) {
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量查询分块索引同步状态。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncStatusRequest {
|
||||||
|
private BigInteger documentId;
|
||||||
|
private List<BigInteger> ids;
|
||||||
|
|
||||||
|
public BigInteger getDocumentId() { return documentId; }
|
||||||
|
public void setDocumentId(BigInteger documentId) { this.documentId = documentId; }
|
||||||
|
public List<BigInteger> getIds() { return ids; }
|
||||||
|
public void setIds(List<BigInteger> ids) { this.ids = ids; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package tech.easyflow.ai.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文档分块检索索引同步任务。
|
||||||
|
*/
|
||||||
|
@Table("tb_document_chunk_sync_task")
|
||||||
|
public class DocumentChunkSyncTask extends DateEntity implements Serializable {
|
||||||
|
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||||
|
private BigInteger id;
|
||||||
|
private BigInteger chunkId;
|
||||||
|
private BigInteger documentId;
|
||||||
|
private BigInteger documentCollectionId;
|
||||||
|
private String vectorCollection;
|
||||||
|
private String operation;
|
||||||
|
private Long syncVersion;
|
||||||
|
private String status;
|
||||||
|
private Integer attemptCount;
|
||||||
|
private Date nextRetryAt;
|
||||||
|
private Date lastDispatchedAt;
|
||||||
|
private String executionToken;
|
||||||
|
private Date leaseUntil;
|
||||||
|
private String errorCode;
|
||||||
|
private String errorMessage;
|
||||||
|
private Date created;
|
||||||
|
private BigInteger createdBy;
|
||||||
|
private Date modified;
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getChunkId() { return chunkId; }
|
||||||
|
public void setChunkId(BigInteger chunkId) { this.chunkId = chunkId; }
|
||||||
|
public BigInteger getDocumentId() { return documentId; }
|
||||||
|
public void setDocumentId(BigInteger documentId) { this.documentId = documentId; }
|
||||||
|
public BigInteger getDocumentCollectionId() { return documentCollectionId; }
|
||||||
|
public void setDocumentCollectionId(BigInteger documentCollectionId) { this.documentCollectionId = documentCollectionId; }
|
||||||
|
public String getVectorCollection() { return vectorCollection; }
|
||||||
|
public void setVectorCollection(String vectorCollection) { this.vectorCollection = vectorCollection; }
|
||||||
|
public String getOperation() { return operation; }
|
||||||
|
public void setOperation(String operation) { this.operation = operation; }
|
||||||
|
public Long getSyncVersion() { return syncVersion; }
|
||||||
|
public void setSyncVersion(Long syncVersion) { this.syncVersion = syncVersion; }
|
||||||
|
public String getStatus() { return status; }
|
||||||
|
public void setStatus(String status) { this.status = status; }
|
||||||
|
public Integer getAttemptCount() { return attemptCount; }
|
||||||
|
public void setAttemptCount(Integer attemptCount) { this.attemptCount = attemptCount; }
|
||||||
|
public Date getNextRetryAt() { return nextRetryAt; }
|
||||||
|
public void setNextRetryAt(Date nextRetryAt) { this.nextRetryAt = nextRetryAt; }
|
||||||
|
public Date getLastDispatchedAt() { return lastDispatchedAt; }
|
||||||
|
public void setLastDispatchedAt(Date lastDispatchedAt) { this.lastDispatchedAt = lastDispatchedAt; }
|
||||||
|
public String getExecutionToken() { return executionToken; }
|
||||||
|
public void setExecutionToken(String executionToken) { this.executionToken = executionToken; }
|
||||||
|
public Date getLeaseUntil() { return leaseUntil; }
|
||||||
|
public void setLeaseUntil(Date leaseUntil) { this.leaseUntil = leaseUntil; }
|
||||||
|
public String getErrorCode() { return errorCode; }
|
||||||
|
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||||
|
public String getErrorMessage() { return errorMessage; }
|
||||||
|
public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
|
||||||
|
@Override public Date getCreated() { return created; }
|
||||||
|
@Override public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override public Date getModified() { return modified; }
|
||||||
|
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import com.easyagents.store.milvus.MilvusVectorStore;
|
|||||||
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
import com.mybatisflex.annotation.Table;
|
import com.mybatisflex.annotation.Table;
|
||||||
import tech.easyflow.ai.config.AiMilvusConfig;
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusClientManager;
|
||||||
import tech.easyflow.ai.chattime.availability.ChatTimeToolAvailabilityContext;
|
import tech.easyflow.ai.chattime.availability.ChatTimeToolAvailabilityContext;
|
||||||
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
|
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
|
||||||
import tech.easyflow.ai.entity.base.DocumentCollectionBase;
|
import tech.easyflow.ai.entity.base.DocumentCollectionBase;
|
||||||
@@ -103,8 +104,10 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
|
|||||||
|
|
||||||
private DocumentStore milvusStore() {
|
private DocumentStore milvusStore() {
|
||||||
AiMilvusConfig aiMilvusConfig = SpringContextUtil.getBean(AiMilvusConfig.class);
|
AiMilvusConfig aiMilvusConfig = SpringContextUtil.getBean(AiMilvusConfig.class);
|
||||||
|
AiMilvusClientManager clientManager = SpringContextUtil.getBean(AiMilvusClientManager.class);
|
||||||
|
clientManager.reconfigureIfNeeded(aiMilvusConfig);
|
||||||
MilvusVectorStoreConfig milvusVectorStoreConfig = aiMilvusConfig.copyForCollection(this.getVectorStoreCollection());
|
MilvusVectorStoreConfig milvusVectorStoreConfig = aiMilvusConfig.copyForCollection(this.getVectorStoreCollection());
|
||||||
return new MilvusVectorStore(milvusVectorStoreConfig);
|
return new MilvusVectorStore(milvusVectorStoreConfig, clientManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Tool toFunction(boolean needEnglishName) {
|
public Tool toFunction(boolean needEnglishName) {
|
||||||
|
|||||||
@@ -46,6 +46,18 @@ public class DocumentChunkBase implements Serializable {
|
|||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展元信息")
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展元信息")
|
||||||
private Map<String, Object> options;
|
private Map<String, Object> options;
|
||||||
|
|
||||||
|
@Column(comment = "检索索引同步状态")
|
||||||
|
private String indexSyncStatus;
|
||||||
|
|
||||||
|
@Column(comment = "检索索引同步版本")
|
||||||
|
private Long indexSyncVersion;
|
||||||
|
|
||||||
|
@Column(comment = "脱敏同步错误码")
|
||||||
|
private String indexSyncErrorCode;
|
||||||
|
|
||||||
|
@Column(comment = "脱敏同步错误摘要")
|
||||||
|
private String indexSyncErrorMessage;
|
||||||
|
|
||||||
public BigInteger getId() {
|
public BigInteger getId() {
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
@@ -94,4 +106,36 @@ public class DocumentChunkBase implements Serializable {
|
|||||||
this.options = options;
|
this.options = options;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getIndexSyncStatus() {
|
||||||
|
return indexSyncStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncStatus(String indexSyncStatus) {
|
||||||
|
this.indexSyncStatus = indexSyncStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getIndexSyncVersion() {
|
||||||
|
return indexSyncVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncVersion(Long indexSyncVersion) {
|
||||||
|
this.indexSyncVersion = indexSyncVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getIndexSyncErrorCode() {
|
||||||
|
return indexSyncErrorCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncErrorCode(String indexSyncErrorCode) {
|
||||||
|
this.indexSyncErrorCode = indexSyncErrorCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getIndexSyncErrorMessage() {
|
||||||
|
return indexSyncErrorMessage;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncErrorMessage(String indexSyncErrorMessage) {
|
||||||
|
this.indexSyncErrorMessage = indexSyncErrorMessage;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ package tech.easyflow.ai.mapper;
|
|||||||
|
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import com.mybatisflex.core.BaseMapper;
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
import org.apache.ibatis.annotations.Update;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 映射层。
|
* 映射层。
|
||||||
@@ -11,4 +17,23 @@ import com.mybatisflex.core.BaseMapper;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentChunkMapper extends BaseMapper<DocumentChunk> {
|
public interface DocumentChunkMapper extends BaseMapper<DocumentChunk> {
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk SET index_sync_status=#{status}, "
|
||||||
|
+ "index_sync_error_code=#{errorCode}, index_sync_error_message=#{errorMessage} "
|
||||||
|
+ "WHERE id=#{id} AND index_sync_version=#{version}")
|
||||||
|
int updateSyncState(@Param("id") BigInteger id,
|
||||||
|
@Param("version") long version,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("errorCode") String errorCode,
|
||||||
|
@Param("errorMessage") String errorMessage);
|
||||||
|
|
||||||
|
@Select("<script>SELECT id, document_id AS documentId, "
|
||||||
|
+ "document_collection_id AS documentCollectionId, "
|
||||||
|
+ "index_sync_status AS indexSyncStatus, index_sync_version AS indexSyncVersion, "
|
||||||
|
+ "index_sync_error_code AS indexSyncErrorCode, "
|
||||||
|
+ "index_sync_error_message AS indexSyncErrorMessage "
|
||||||
|
+ "FROM tb_document_chunk WHERE document_id=#{documentId} AND id IN "
|
||||||
|
+ "<foreach collection='ids' item='id' open='(' separator=',' close=')'>#{id}</foreach>"
|
||||||
|
+ "</script>")
|
||||||
|
List<DocumentChunk> selectSyncStates(@Param("documentId") BigInteger documentId,
|
||||||
|
@Param("ids") List<BigInteger> ids);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package tech.easyflow.ai.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
import org.apache.ibatis.annotations.Update;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文档分块索引同步任务映射层。
|
||||||
|
*/
|
||||||
|
public interface DocumentChunkSyncTaskMapper extends BaseMapper<DocumentChunkSyncTask> {
|
||||||
|
|
||||||
|
String SELECT_COLUMNS = "id, chunk_id AS chunkId, document_id AS documentId, "
|
||||||
|
+ "document_collection_id AS documentCollectionId, "
|
||||||
|
+ "vector_collection AS vectorCollection, operation, sync_version AS syncVersion, "
|
||||||
|
+ "status, attempt_count AS attemptCount, next_retry_at AS nextRetryAt, "
|
||||||
|
+ "last_dispatched_at AS lastDispatchedAt, execution_token AS executionToken, "
|
||||||
|
+ "lease_until AS leaseUntil, error_code AS errorCode, error_message AS errorMessage, "
|
||||||
|
+ "created, created_by AS createdBy, modified, modified_by AS modifiedBy";
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='SUPERSEDED', "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, modified=#{now} "
|
||||||
|
+ "WHERE chunk_id=#{chunkId} AND sync_version < #{syncVersion} "
|
||||||
|
+ "AND status IN ('PENDING','RUNNING','FAILED')")
|
||||||
|
int supersedeOlder(@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("syncVersion") long syncVersion,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='SUPERSEDED', "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, modified=#{now} "
|
||||||
|
+ "WHERE chunk_id=#{chunkId} AND status IN ('PENDING','RUNNING','FAILED')")
|
||||||
|
int supersedeChunk(@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 锁定同一分块的任务版本范围,保证外部索引写入与新版本建单按版本串行。
|
||||||
|
*/
|
||||||
|
@Select("SELECT id FROM tb_document_chunk_sync_task WHERE chunk_id=#{chunkId} "
|
||||||
|
+ "ORDER BY sync_version, id FOR UPDATE")
|
||||||
|
List<BigInteger> lockChunkTasks(@Param("chunkId") BigInteger chunkId);
|
||||||
|
|
||||||
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task "
|
||||||
|
+ "WHERE status='PENDING' AND next_retry_at <= #{now} "
|
||||||
|
+ "AND (last_dispatched_at IS NULL OR last_dispatched_at <= #{redispatchBefore}) "
|
||||||
|
+ "ORDER BY next_retry_at, id LIMIT #{limit}")
|
||||||
|
List<DocumentChunkSyncTask> selectPendingDue(@Param("now") Date now,
|
||||||
|
@Param("redispatchBefore") Date redispatchBefore,
|
||||||
|
@Param("limit") int limit);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET last_dispatched_at=#{now}, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='PENDING' AND next_retry_at <= #{now} "
|
||||||
|
+ "AND (last_dispatched_at IS NULL OR last_dispatched_at <= #{redispatchBefore})")
|
||||||
|
int markDispatched(@Param("id") BigInteger id,
|
||||||
|
@Param("now") Date now,
|
||||||
|
@Param("redispatchBefore") Date redispatchBefore);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='PENDING', "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, next_retry_at=#{now}, "
|
||||||
|
+ "last_dispatched_at=NULL, modified=#{now} "
|
||||||
|
+ "WHERE status='RUNNING' AND lease_until <= #{now}")
|
||||||
|
int recoverExpired(@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='RUNNING', "
|
||||||
|
+ "attempt_count=attempt_count + 1, execution_token=#{token}, "
|
||||||
|
+ "lease_until=#{leaseUntil}, error_code=NULL, error_message=NULL, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='PENDING' AND next_retry_at <= #{now}")
|
||||||
|
int claim(@Param("id") BigInteger id,
|
||||||
|
@Param("token") String token,
|
||||||
|
@Param("leaseUntil") Date leaseUntil,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status=#{status}, "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, error_code=#{errorCode}, "
|
||||||
|
+ "error_message=#{errorMessage}, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='RUNNING' AND execution_token=#{token}")
|
||||||
|
int finishOwned(@Param("id") BigInteger id,
|
||||||
|
@Param("token") String token,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("errorCode") String errorCode,
|
||||||
|
@Param("errorMessage") String errorMessage,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status=#{status}, "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, next_retry_at=#{nextRetryAt}, "
|
||||||
|
+ "last_dispatched_at=NULL, error_code=#{errorCode}, "
|
||||||
|
+ "error_message=#{errorMessage}, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='RUNNING' AND execution_token=#{token}")
|
||||||
|
int failOrRetryOwned(@Param("id") BigInteger id,
|
||||||
|
@Param("token") String token,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("nextRetryAt") Date nextRetryAt,
|
||||||
|
@Param("errorCode") String errorCode,
|
||||||
|
@Param("errorMessage") String errorMessage,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='PENDING', "
|
||||||
|
+ "attempt_count=0, next_retry_at=#{now}, last_dispatched_at=NULL, "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, error_code=NULL, error_message=NULL, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND chunk_id=#{chunkId} AND sync_version=#{syncVersion} "
|
||||||
|
+ "AND status='FAILED'")
|
||||||
|
int retryFailed(@Param("id") BigInteger id,
|
||||||
|
@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("syncVersion") long syncVersion,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task WHERE chunk_id=#{chunkId} "
|
||||||
|
+ "AND sync_version=#{syncVersion} AND status='FAILED' ORDER BY id DESC LIMIT 1")
|
||||||
|
DocumentChunkSyncTask selectFailed(@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("syncVersion") long syncVersion);
|
||||||
|
}
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
package tech.easyflow.ai.service;
|
package tech.easyflow.ai.service;
|
||||||
|
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
|
||||||
import com.mybatisflex.core.service.IService;
|
import com.mybatisflex.core.service.IService;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层。
|
* 服务层。
|
||||||
@@ -14,5 +16,36 @@ import java.math.BigInteger;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentChunkService extends IService<DocumentChunk> {
|
public interface DocumentChunkService extends IService<DocumentChunk> {
|
||||||
|
|
||||||
boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId);
|
/**
|
||||||
|
* 更新分块的 Markdown 正文,并创建后台检索索引同步任务。
|
||||||
|
*
|
||||||
|
* @param knowledgeId 知识库 ID
|
||||||
|
* @param chunkId 分块 ID
|
||||||
|
* @param markdown Markdown 正文
|
||||||
|
* @return 更新后的分块
|
||||||
|
*/
|
||||||
|
DocumentChunk updateContent(BigInteger knowledgeId, BigInteger chunkId, String markdown);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除分块,并创建后台检索索引清理任务。
|
||||||
|
*
|
||||||
|
* @param knowledgeId 知识库 ID
|
||||||
|
* @param chunkId 分块 ID
|
||||||
|
* @return 删除结果
|
||||||
|
*/
|
||||||
|
DocumentChunkDeleteResult deleteChunk(BigInteger knowledgeId, BigInteger chunkId);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 手动重试当前版本的检索索引同步。
|
||||||
|
*/
|
||||||
|
DocumentChunk retryIndexSync(BigInteger knowledgeId, BigInteger chunkId, long syncVersion);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量读取文档内分块的索引同步状态。
|
||||||
|
*/
|
||||||
|
List<DocumentChunkSyncStatus> listIndexSyncStatus(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger documentId,
|
||||||
|
List<BigInteger> chunkIds
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,36 +1,365 @@
|
|||||||
package tech.easyflow.ai.service.impl;
|
package tech.easyflow.ai.service.impl;
|
||||||
|
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.rag.core.BgeM3ChunkSafety;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import com.easyagents.rag.core.RagDefaults;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
|
||||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncState;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncTaskAppService;
|
||||||
|
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.entity.Document;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层实现。
|
* 分块服务层实现。
|
||||||
*
|
|
||||||
* @author michael
|
|
||||||
* @since 2024-08-23
|
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
public class DocumentChunkServiceImpl extends ServiceImpl<DocumentChunkMapper, DocumentChunk> implements DocumentChunkService {
|
public class DocumentChunkServiceImpl
|
||||||
|
extends ServiceImpl<DocumentChunkMapper, DocumentChunk>
|
||||||
|
implements DocumentChunkService {
|
||||||
|
|
||||||
@Autowired
|
public static final int DOCUMENT_CHUNK_EMPTY_REQUIRES_DELETE = 42901;
|
||||||
private SearcherFactory searcherFactory;
|
public static final int DOCUMENT_CHUNK_LOCK_UNAVAILABLE = 42905;
|
||||||
|
|
||||||
|
private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(5);
|
||||||
|
private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(30);
|
||||||
|
private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile(
|
||||||
|
"!\\[([^\\]]*)\\]\\((?:[^()\\r\\n]|\\([^()\\r\\n]*\\))*\\)"
|
||||||
|
);
|
||||||
|
private static final Pattern HTML_IMAGE_PATTERN = Pattern.compile(
|
||||||
|
"<img\\b[^>]*>",
|
||||||
|
Pattern.CASE_INSENSITIVE
|
||||||
|
);
|
||||||
|
private static final Pattern HTML_ALT_PATTERN = Pattern.compile(
|
||||||
|
"\\balt\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')",
|
||||||
|
Pattern.CASE_INSENSITIVE
|
||||||
|
);
|
||||||
|
|
||||||
|
private final DocumentChunkMapper documentChunkMapper;
|
||||||
|
private final DocumentMapper documentMapper;
|
||||||
|
private final DocumentCollectionService documentCollectionService;
|
||||||
|
private final DocumentChunkSyncTaskAppService syncTaskAppService;
|
||||||
|
private final TransactionTemplate transactionTemplate;
|
||||||
|
private final RedisLockExecutor redisLockExecutor;
|
||||||
|
|
||||||
|
public DocumentChunkServiceImpl(
|
||||||
|
DocumentChunkMapper documentChunkMapper,
|
||||||
|
DocumentMapper documentMapper,
|
||||||
|
DocumentCollectionService documentCollectionService,
|
||||||
|
DocumentChunkSyncTaskAppService syncTaskAppService,
|
||||||
|
PlatformTransactionManager transactionManager,
|
||||||
|
RedisLockExecutor redisLockExecutor
|
||||||
|
) {
|
||||||
|
this.documentChunkMapper = documentChunkMapper;
|
||||||
|
this.documentMapper = documentMapper;
|
||||||
|
this.documentCollectionService = documentCollectionService;
|
||||||
|
this.syncTaskAppService = syncTaskAppService;
|
||||||
|
this.transactionTemplate = new TransactionTemplate(transactionManager);
|
||||||
|
this.redisLockExecutor = redisLockExecutor;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId) {
|
public DocumentChunk updateContent(
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher();
|
BigInteger knowledgeId,
|
||||||
// 删除搜索引擎中的数据
|
BigInteger chunkId,
|
||||||
if (searcher == null){
|
String markdown
|
||||||
return true;
|
) {
|
||||||
|
if (markdown == null || markdown.trim().isEmpty()) {
|
||||||
|
throw new BusinessException(
|
||||||
|
422,
|
||||||
|
DOCUMENT_CHUNK_EMPTY_REQUIRES_DELETE,
|
||||||
|
"分块内容为空,请删除该分块"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
return searcher.deleteDocument(chunkId);
|
DocumentChunk snapshot = requireChunk(knowledgeId, chunkId);
|
||||||
|
UpdateOutcome outcome = withDocumentLock(
|
||||||
|
snapshot.getDocumentId(),
|
||||||
|
() -> updateContentLocked(knowledgeId, chunkId, markdown)
|
||||||
|
);
|
||||||
|
syncTaskAppService.dispatchBestEffort(outcome.taskId());
|
||||||
|
return outcome.chunk();
|
||||||
|
}
|
||||||
|
|
||||||
|
private UpdateOutcome updateContentLocked(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
String markdown
|
||||||
|
) {
|
||||||
|
DocumentChunk current = requireChunk(knowledgeId, chunkId);
|
||||||
|
DocumentCollection collection = requireKnowledge(knowledgeId);
|
||||||
|
String searchableContent = toSearchableContent(markdown);
|
||||||
|
assertWithinEmbeddingLimit(chunkId, searchableContent);
|
||||||
|
Map<String, Object> options = current.getOptions() == null
|
||||||
|
? new HashMap<>()
|
||||||
|
: new HashMap<>(current.getOptions());
|
||||||
|
options.put(DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN, markdown);
|
||||||
|
long nextVersion = current.getIndexSyncVersion() == null
|
||||||
|
? 1L
|
||||||
|
: current.getIndexSyncVersion() + 1L;
|
||||||
|
|
||||||
|
UpdateOutcome outcome = transactionTemplate.execute(status -> {
|
||||||
|
DocumentChunkSyncTask task = syncTaskAppService.createTask(
|
||||||
|
current,
|
||||||
|
collection,
|
||||||
|
DocumentChunkSyncState.OPERATION_UPSERT,
|
||||||
|
nextVersion,
|
||||||
|
new Date()
|
||||||
|
);
|
||||||
|
DocumentChunk update = new DocumentChunk();
|
||||||
|
update.setId(chunkId);
|
||||||
|
update.setContent(searchableContent);
|
||||||
|
update.setOptions(options);
|
||||||
|
update.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
update.setIndexSyncVersion(nextVersion);
|
||||||
|
update.setIndexSyncErrorCode(null);
|
||||||
|
update.setIndexSyncErrorMessage(null);
|
||||||
|
if (documentChunkMapper.update(update) <= 0) {
|
||||||
|
throw new BusinessException("分块更新失败");
|
||||||
|
}
|
||||||
|
touchDocument(current.getDocumentId(), null);
|
||||||
|
current.setContent(searchableContent);
|
||||||
|
current.setOptions(options);
|
||||||
|
current.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
current.setIndexSyncVersion(nextVersion);
|
||||||
|
current.setIndexSyncErrorCode(null);
|
||||||
|
current.setIndexSyncErrorMessage(null);
|
||||||
|
return new UpdateOutcome(current, task.getId());
|
||||||
|
});
|
||||||
|
if (outcome == null) {
|
||||||
|
throw new BusinessException("分块更新失败");
|
||||||
|
}
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DocumentChunkDeleteResult deleteChunk(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
DocumentChunk snapshot = requireChunk(knowledgeId, chunkId);
|
||||||
|
DeleteOutcome outcome = withDocumentLock(
|
||||||
|
snapshot.getDocumentId(),
|
||||||
|
() -> deleteChunkLocked(knowledgeId, chunkId)
|
||||||
|
);
|
||||||
|
syncTaskAppService.dispatchBestEffort(outcome.taskId());
|
||||||
|
return outcome.result();
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T> T withDocumentLock(BigInteger documentId, Supplier<T> action) {
|
||||||
|
RedisLockExecutor.LockHandle handle;
|
||||||
|
try {
|
||||||
|
handle = redisLockExecutor.tryAcquire(
|
||||||
|
DocumentChunkSyncState.parentLockKey(documentId),
|
||||||
|
LOCK_WAIT_TIMEOUT,
|
||||||
|
LOCK_LEASE_TIMEOUT
|
||||||
|
);
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new BusinessException(
|
||||||
|
503,
|
||||||
|
DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
"分块正在处理中,请稍后重试",
|
||||||
|
exception
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (handle == null) {
|
||||||
|
throw new BusinessException(
|
||||||
|
503,
|
||||||
|
DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
"分块正在处理中,请稍后重试"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
try (handle) {
|
||||||
|
return action.get();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DeleteOutcome deleteChunkLocked(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
DocumentChunk current = requireChunk(knowledgeId, chunkId);
|
||||||
|
DocumentCollection collection = requireKnowledge(knowledgeId);
|
||||||
|
long nextVersion = current.getIndexSyncVersion() == null
|
||||||
|
? 1L
|
||||||
|
: current.getIndexSyncVersion() + 1L;
|
||||||
|
DeleteOutcome outcome = transactionTemplate.execute(status -> {
|
||||||
|
DocumentChunkSyncTask task = syncTaskAppService.createTask(
|
||||||
|
current,
|
||||||
|
collection,
|
||||||
|
DocumentChunkSyncState.OPERATION_DELETE,
|
||||||
|
nextVersion,
|
||||||
|
new Date()
|
||||||
|
);
|
||||||
|
if (documentChunkMapper.deleteById(chunkId) <= 0) {
|
||||||
|
throw new BusinessException("分块删除失败");
|
||||||
|
}
|
||||||
|
long remaining = documentChunkMapper.selectCountByQuery(
|
||||||
|
QueryWrapper.create().eq(DocumentChunk::getDocumentId, current.getDocumentId())
|
||||||
|
);
|
||||||
|
touchDocument(current.getDocumentId(), remaining);
|
||||||
|
return new DeleteOutcome(
|
||||||
|
new DocumentChunkDeleteResult(chunkId, current.getDocumentId(), remaining),
|
||||||
|
task.getId()
|
||||||
|
);
|
||||||
|
});
|
||||||
|
if (outcome == null) {
|
||||||
|
throw new BusinessException("分块删除失败");
|
||||||
|
}
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DocumentChunk retryIndexSync(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
long syncVersion
|
||||||
|
) {
|
||||||
|
requireChunk(knowledgeId, chunkId);
|
||||||
|
return syncTaskAppService.retry(chunkId, syncVersion);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DocumentChunkSyncStatus> listIndexSyncStatus(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger documentId,
|
||||||
|
List<BigInteger> chunkIds
|
||||||
|
) {
|
||||||
|
if (documentId == null || chunkIds == null || chunkIds.isEmpty()) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
BigInteger effectiveKnowledgeId = knowledgeId;
|
||||||
|
if (effectiveKnowledgeId == null) {
|
||||||
|
Document document = documentMapper.selectOneById(documentId);
|
||||||
|
if (document == null || document.getCollectionId() == null) {
|
||||||
|
throw new BusinessException("文档不存在");
|
||||||
|
}
|
||||||
|
effectiveKnowledgeId = document.getCollectionId();
|
||||||
|
}
|
||||||
|
if (chunkIds.size() > 200) {
|
||||||
|
throw new BusinessException("单次最多查询 200 个分块状态");
|
||||||
|
}
|
||||||
|
List<DocumentChunk> chunks = documentChunkMapper.selectSyncStates(documentId, chunkIds);
|
||||||
|
for (DocumentChunk chunk : chunks) {
|
||||||
|
if (chunk.getDocumentCollectionId() == null
|
||||||
|
|| chunk.getDocumentCollectionId().compareTo(effectiveKnowledgeId) != 0) {
|
||||||
|
throw new BusinessException("分块不存在");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return chunks.stream().map(chunk -> new DocumentChunkSyncStatus(
|
||||||
|
chunk.getId(),
|
||||||
|
chunk.getIndexSyncStatus(),
|
||||||
|
chunk.getIndexSyncVersion(),
|
||||||
|
chunk.getIndexSyncErrorCode(),
|
||||||
|
chunk.getIndexSyncErrorMessage()
|
||||||
|
)).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DocumentChunk requireChunk(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
if (knowledgeId == null || chunkId == null) {
|
||||||
|
throw new BusinessException("分块不存在");
|
||||||
|
}
|
||||||
|
DocumentChunk chunk = documentChunkMapper.selectOneById(chunkId);
|
||||||
|
if (chunk == null || chunk.getDocumentCollectionId() == null
|
||||||
|
|| chunk.getDocumentCollectionId().compareTo(knowledgeId) != 0) {
|
||||||
|
throw new BusinessException("分块不存在");
|
||||||
|
}
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DocumentCollection requireKnowledge(BigInteger knowledgeId) {
|
||||||
|
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
|
||||||
|
if (knowledge == null) {
|
||||||
|
throw new BusinessException("知识库不存在");
|
||||||
|
}
|
||||||
|
return knowledge;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void touchDocument(BigInteger documentId, Long chunkCount) {
|
||||||
|
if (documentId == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Document update = new Document();
|
||||||
|
update.setId(documentId);
|
||||||
|
update.setModified(new Date());
|
||||||
|
if (chunkCount != null) {
|
||||||
|
int count = chunkCount > Integer.MAX_VALUE
|
||||||
|
? Integer.MAX_VALUE
|
||||||
|
: chunkCount.intValue();
|
||||||
|
update.setTotalChunks(count);
|
||||||
|
update.setCompletedChunks(count);
|
||||||
|
update.setFailedChunks(0);
|
||||||
|
update.setProgressPercent(100);
|
||||||
|
}
|
||||||
|
if (documentMapper.update(update) <= 0) {
|
||||||
|
throw new BusinessException("文档状态更新失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertWithinEmbeddingLimit(BigInteger chunkId, String content) {
|
||||||
|
int tokenEstimate = BgeM3ChunkSafety.estimateContentTokens(content);
|
||||||
|
if (tokenEstimate > RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT) {
|
||||||
|
throw new BusinessException(
|
||||||
|
"分块内容超过向量模型上下文上限,请缩短后保存:chunkId=" + chunkId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String toSearchableContent(String markdown) {
|
||||||
|
return replaceHtmlImages(replaceMarkdownImages(markdown)).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String replaceMarkdownImages(String markdown) {
|
||||||
|
Matcher matcher = MARKDOWN_IMAGE_PATTERN.matcher(markdown);
|
||||||
|
StringBuffer output = new StringBuffer();
|
||||||
|
while (matcher.find()) {
|
||||||
|
matcher.appendReplacement(
|
||||||
|
output,
|
||||||
|
Matcher.quoteReplacement(imageSearchText(matcher.group(1)))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
matcher.appendTail(output);
|
||||||
|
return output.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String replaceHtmlImages(String content) {
|
||||||
|
Matcher matcher = HTML_IMAGE_PATTERN.matcher(content);
|
||||||
|
StringBuffer output = new StringBuffer();
|
||||||
|
while (matcher.find()) {
|
||||||
|
Matcher altMatcher = HTML_ALT_PATTERN.matcher(matcher.group());
|
||||||
|
String alt = altMatcher.find()
|
||||||
|
? (altMatcher.group(1) == null ? altMatcher.group(2) : altMatcher.group(1))
|
||||||
|
: null;
|
||||||
|
matcher.appendReplacement(output, Matcher.quoteReplacement(imageSearchText(alt)));
|
||||||
|
}
|
||||||
|
matcher.appendTail(output);
|
||||||
|
return output.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String imageSearchText(String alt) {
|
||||||
|
return alt == null || alt.trim().isEmpty() ? "图片" : alt.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private record UpdateOutcome(DocumentChunk chunk, BigInteger taskId) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record DeleteOutcome(DocumentChunkDeleteResult result, BigInteger taskId) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -648,6 +648,7 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
QueryWrapper chunkWrapper = QueryWrapper.create();
|
QueryWrapper chunkWrapper = QueryWrapper.create();
|
||||||
chunkWrapper.in(DocumentChunk::getId, chunkIds);
|
chunkWrapper.in(DocumentChunk::getId, chunkIds);
|
||||||
chunkWrapper.eq(DocumentChunk::getDocumentCollectionId, documentCollection.getId());
|
chunkWrapper.eq(DocumentChunk::getDocumentCollectionId, documentCollection.getId());
|
||||||
|
chunkWrapper.eq(DocumentChunk::getIndexSyncStatus, "SYNCED");
|
||||||
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByQuery(chunkWrapper).stream()
|
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByQuery(chunkWrapper).stream()
|
||||||
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
||||||
if (chunkMap.isEmpty()) {
|
if (chunkMap.isEmpty()) {
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncState;
|
||||||
import tech.easyflow.common.util.SearchKeywordUtil;
|
import tech.easyflow.common.util.SearchKeywordUtil;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||||
@@ -40,6 +41,7 @@ import tech.easyflow.ai.documentimport.task.KnowledgeDocumentImportTaskAppServic
|
|||||||
import tech.easyflow.ai.entity.*;
|
import tech.easyflow.ai.entity.*;
|
||||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
@@ -47,6 +49,7 @@ import tech.easyflow.ai.service.DocumentService;
|
|||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
||||||
import tech.easyflow.common.ai.rag.ExcelDocumentSplitter;
|
import tech.easyflow.common.ai.rag.ExcelDocumentSplitter;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
import tech.easyflow.common.util.FileUtil;
|
import tech.easyflow.common.util.FileUtil;
|
||||||
@@ -58,8 +61,11 @@ import java.io.IOException;
|
|||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
||||||
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
||||||
@@ -83,6 +89,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
@Resource
|
@Resource
|
||||||
private DocumentChunkMapper documentChunkMapper;
|
private DocumentChunkMapper documentChunkMapper;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DocumentChunkSyncTaskMapper documentChunkSyncTaskMapper;
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
private DocumentCollectionService knowledgeService;
|
private DocumentCollectionService knowledgeService;
|
||||||
|
|
||||||
@@ -110,6 +119,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
@Autowired
|
@Autowired
|
||||||
private KnowledgeDocumentImportTaskAppService importTaskAppService;
|
private KnowledgeDocumentImportTaskAppService importTaskAppService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private RedisLockExecutor redisLockExecutor;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
|
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
|
||||||
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null, false);
|
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null, false);
|
||||||
@@ -202,6 +214,49 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
@Override
|
@Override
|
||||||
@Transactional
|
@Transactional
|
||||||
public boolean removeDoc(String id) {
|
public boolean removeDoc(String id) {
|
||||||
|
AtomicBoolean removalCompleted = new AtomicBoolean();
|
||||||
|
AtomicReference<Boolean> removalResult = new AtomicReference<>();
|
||||||
|
try {
|
||||||
|
return redisLockExecutor.executeWithRenewingLock(
|
||||||
|
DocumentChunkSyncState.parentLockKey(id),
|
||||||
|
Duration.ofSeconds(5),
|
||||||
|
Duration.ofSeconds(30),
|
||||||
|
() -> {
|
||||||
|
try {
|
||||||
|
boolean result = removeDocLocked(id);
|
||||||
|
removalResult.set(result);
|
||||||
|
removalCompleted.set(true);
|
||||||
|
return result;
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new DocumentRemovalExecutionException(exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (DocumentRemovalExecutionException exception) {
|
||||||
|
throw exception.getOriginalCause();
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
if (removalCompleted.get()) {
|
||||||
|
Log.warn(
|
||||||
|
"文档删除已完成但分布式锁在收尾阶段失效,继续提交数据库事务: documentId={}",
|
||||||
|
id,
|
||||||
|
exception
|
||||||
|
);
|
||||||
|
return Boolean.TRUE.equals(removalResult.get());
|
||||||
|
}
|
||||||
|
throw new BusinessException(
|
||||||
|
503,
|
||||||
|
DocumentChunkServiceImpl.DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
"文档正在处理中,请稍后重试",
|
||||||
|
exception
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean removeDocLocked(String id) {
|
||||||
// 查询该文档对应哪些分割的字段,先删除
|
// 查询该文档对应哪些分割的字段,先删除
|
||||||
QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id);
|
QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id);
|
||||||
Document oneByQuery = documentMapper.selectOneByQuery(queryWrapperDocument);
|
Document oneByQuery = documentMapper.selectOneByQuery(queryWrapperDocument);
|
||||||
@@ -218,23 +273,38 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
|
|
||||||
QueryWrapper queryWrapper = QueryWrapper.create()
|
QueryWrapper queryWrapper = QueryWrapper.create()
|
||||||
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
|
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
|
||||||
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(
|
List<BigInteger> chunkIds = new ArrayList<>(
|
||||||
queryWrapper,
|
documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class)
|
||||||
BigInteger.class
|
|
||||||
);
|
);
|
||||||
|
chunkIds.sort(Comparator.naturalOrder());
|
||||||
DocumentStore documentStore = null;
|
DocumentStore documentStore = null;
|
||||||
try {
|
try {
|
||||||
|
Model model = null;
|
||||||
if (!chunkIds.isEmpty()) {
|
if (!chunkIds.isEmpty()) {
|
||||||
documentStore = knowledge.toDocumentStore();
|
documentStore = knowledge.toDocumentStore();
|
||||||
if (documentStore == null) {
|
if (documentStore == null) {
|
||||||
return false;
|
throw new BusinessException("文档向量存储不可用");
|
||||||
}
|
}
|
||||||
Model model = modelService.getById(
|
model = modelService.getById(
|
||||||
knowledge.getVectorEmbedModelId()
|
knowledge.getVectorEmbedModelId()
|
||||||
);
|
);
|
||||||
if (model == null) {
|
if (model == null) {
|
||||||
return false;
|
throw new BusinessException("文档向量模型不存在");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
Date supersededAt = new Date();
|
||||||
|
for (BigInteger chunkId : chunkIds) {
|
||||||
|
redisLockExecutor.executeWithRenewingLock(
|
||||||
|
DocumentChunkSyncState.syncLockKey(chunkId),
|
||||||
|
Duration.ofSeconds(5),
|
||||||
|
Duration.ofSeconds(30),
|
||||||
|
() -> {
|
||||||
|
documentChunkSyncTaskMapper.supersedeChunk(chunkId, supersededAt);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!chunkIds.isEmpty()) {
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(
|
StoreOptions options = StoreOptions.ofCollectionName(
|
||||||
knowledge.getVectorStoreCollection()
|
knowledge.getVectorStoreCollection()
|
||||||
);
|
);
|
||||||
@@ -253,7 +323,16 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
// 删除搜索引擎中的数据
|
// 删除搜索引擎中的数据
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher();
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
if (searcher != null) {
|
if (searcher != null) {
|
||||||
chunkIds.forEach(searcher::deleteDocument);
|
for (BigInteger chunkId : chunkIds) {
|
||||||
|
if (!searcher.deleteDocument(chunkId)) {
|
||||||
|
Log.error(
|
||||||
|
"删除文档关键词索引失败: documentId={}, chunkId={}",
|
||||||
|
id,
|
||||||
|
chunkId
|
||||||
|
);
|
||||||
|
throw new BusinessException("文档关键词索引删除失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
|
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
|
||||||
if (ck < 0) {
|
if (ck < 0) {
|
||||||
@@ -1099,6 +1178,17 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static final class DocumentRemovalExecutionException extends RuntimeException {
|
||||||
|
|
||||||
|
private DocumentRemovalExecutionException(RuntimeException cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
private RuntimeException getOriginalCause() {
|
||||||
|
return (RuntimeException) getCause();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public DocumentSplitter getDocumentSplitter(DocumentCollectionSplitParams params) {
|
public DocumentSplitter getDocumentSplitter(DocumentCollectionSplitParams params) {
|
||||||
String splitterName = params.getSplitterName();
|
String splitterName = params.getSplitterName();
|
||||||
int chunkSize = params.getChunkSize();
|
int chunkSize = params.getChunkSize();
|
||||||
|
|||||||
@@ -0,0 +1,233 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.mockito.InOrder;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.SimpleTransactionStatus;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.ai.service.ModelService;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步任务恢复与版本语义测试。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncTaskAppServiceTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void dispatchPendingShouldRecoverExpiredTasksAndSendClaimedRows() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
Mockito.when(fixture.taskMapper.selectPendingDue(
|
||||||
|
Mockito.any(), Mockito.any(), Mockito.anyInt()
|
||||||
|
)).thenReturn(List.of(fixture.task));
|
||||||
|
Mockito.when(fixture.taskMapper.markDispatched(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
|
||||||
|
fixture.service.dispatchPendingTasks();
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).recoverExpired(Mockito.any());
|
||||||
|
Mockito.verify(fixture.producer).send(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void outdatedTaskShouldBeSupersededWithoutTouchingIndexes() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
fixture.chunk.setIndexSyncVersion(2L);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).finishOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.TASK_SUPERSEDED),
|
||||||
|
Mockito.isNull(), Mockito.isNull(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verifyNoInteractions(fixture.collectionService);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void transientFailureShouldReturnTaskToPending() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
InOrder order = Mockito.inOrder(
|
||||||
|
fixture.taskMapper,
|
||||||
|
fixture.collectionService
|
||||||
|
);
|
||||||
|
order.verify(fixture.taskMapper).lockChunkTasks(fixture.chunkId);
|
||||||
|
order.verify(fixture.collectionService).getById(Mockito.any());
|
||||||
|
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.PENDING), Mockito.any(),
|
||||||
|
Mockito.eq("INDEX_SOURCE_MISSING"), Mockito.anyString(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verify(fixture.chunkMapper, Mockito.never()).updateSyncState(
|
||||||
|
Mockito.any(), Mockito.anyLong(), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void workerShouldLockChunkTaskRangeBeforeReadingOwnedTask() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
InOrder order = Mockito.inOrder(fixture.taskMapper);
|
||||||
|
order.verify(fixture.taskMapper).selectOneById(fixture.taskId);
|
||||||
|
order.verify(fixture.taskMapper).claim(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
);
|
||||||
|
order.verify(fixture.taskMapper).lockChunkTasks(fixture.chunkId);
|
||||||
|
order.verify(fixture.taskMapper).selectOneById(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void exhaustedFailureShouldExposeFailedStateOnCurrentVersion() {
|
||||||
|
Fixture fixture = fixture(5);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.FAILED), Mockito.any(),
|
||||||
|
Mockito.eq("INDEX_SOURCE_MISSING"), Mockito.anyString(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verify(fixture.chunkMapper).updateSyncState(
|
||||||
|
fixture.chunkId, 1L, DocumentChunkSyncState.FAILED,
|
||||||
|
"INDEX_SOURCE_MISSING", "分块或知识库不存在"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void exhaustedDeleteFailureShouldRemainPendingForDurableCleanup() {
|
||||||
|
Fixture fixture = fixture(5);
|
||||||
|
fixture.task.setOperation(DocumentChunkSyncState.OPERATION_DELETE);
|
||||||
|
Mockito.when(fixture.chunkMapper.selectOneById(fixture.chunkId)).thenReturn(null);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.PENDING), Mockito.any(),
|
||||||
|
Mockito.eq("INDEX_SYNC_FAILED"), Mockito.anyString(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verify(fixture.chunkMapper, Mockito.never()).updateSyncState(
|
||||||
|
Mockito.any(), Mockito.anyLong(), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void manualRetryShouldResetTaskAndChunkBeforeDispatch() {
|
||||||
|
Fixture fixture = fixture(5);
|
||||||
|
fixture.chunk.setIndexSyncStatus(DocumentChunkSyncState.FAILED);
|
||||||
|
fixture.task.setStatus(DocumentChunkSyncState.FAILED);
|
||||||
|
Mockito.when(fixture.taskMapper.selectFailed(fixture.chunkId, 1L))
|
||||||
|
.thenReturn(fixture.task);
|
||||||
|
Mockito.when(fixture.taskMapper.retryFailed(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.eq(fixture.chunkId),
|
||||||
|
Mockito.eq(1L), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
Mockito.when(fixture.chunkMapper.updateSyncState(
|
||||||
|
fixture.chunkId, 1L, DocumentChunkSyncState.PENDING, null, null
|
||||||
|
)).thenReturn(1);
|
||||||
|
|
||||||
|
fixture.service.retry(fixture.chunkId, 1L);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.producer).send(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Fixture fixture(int attemptCount) {
|
||||||
|
BigInteger taskId = BigInteger.valueOf(10);
|
||||||
|
BigInteger chunkId = BigInteger.valueOf(20);
|
||||||
|
DocumentChunkSyncTask task = new DocumentChunkSyncTask();
|
||||||
|
task.setId(taskId);
|
||||||
|
task.setChunkId(chunkId);
|
||||||
|
task.setDocumentId(BigInteger.valueOf(30));
|
||||||
|
task.setDocumentCollectionId(BigInteger.valueOf(40));
|
||||||
|
task.setVectorCollection("kb-test");
|
||||||
|
task.setOperation(DocumentChunkSyncState.OPERATION_UPSERT);
|
||||||
|
task.setSyncVersion(1L);
|
||||||
|
task.setStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
task.setAttemptCount(attemptCount);
|
||||||
|
|
||||||
|
DocumentChunk chunk = new DocumentChunk();
|
||||||
|
chunk.setId(chunkId);
|
||||||
|
chunk.setDocumentCollectionId(task.getDocumentCollectionId());
|
||||||
|
chunk.setIndexSyncVersion(1L);
|
||||||
|
chunk.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper = Mockito.mock(DocumentChunkSyncTaskMapper.class);
|
||||||
|
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
|
||||||
|
Mockito.when(taskMapper.claim(
|
||||||
|
Mockito.eq(taskId), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
)).thenAnswer(invocation -> {
|
||||||
|
task.setStatus(DocumentChunkSyncState.TASK_RUNNING);
|
||||||
|
task.setExecutionToken(invocation.getArgument(1));
|
||||||
|
return 1;
|
||||||
|
});
|
||||||
|
Mockito.when(taskMapper.lockChunkTasks(chunkId)).thenReturn(List.of(taskId));
|
||||||
|
Mockito.when(taskMapper.failOrRetryOwned(
|
||||||
|
Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
|
||||||
|
Mockito.any(), Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
Mockito.when(taskMapper.finishOwned(
|
||||||
|
Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
|
||||||
|
Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
|
||||||
|
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||||
|
Mockito.when(chunkMapper.selectOneById(chunkId)).thenReturn(chunk);
|
||||||
|
DocumentCollectionService collectionService = Mockito.mock(DocumentCollectionService.class);
|
||||||
|
ModelService modelService = Mockito.mock(ModelService.class);
|
||||||
|
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
|
||||||
|
DocumentChunkSyncTaskProducer producer = Mockito.mock(DocumentChunkSyncTaskProducer.class);
|
||||||
|
PlatformTransactionManager transactionManager = Mockito.mock(PlatformTransactionManager.class);
|
||||||
|
Mockito.when(transactionManager.getTransaction(Mockito.any()))
|
||||||
|
.thenReturn(new SimpleTransactionStatus());
|
||||||
|
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||||
|
Mockito.doAnswer(invocation -> invocation.<Supplier<?>>getArgument(3).get())
|
||||||
|
.when(redisLockExecutor).executeWithRenewingLock(
|
||||||
|
Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.<Supplier<?>>any()
|
||||||
|
);
|
||||||
|
|
||||||
|
DocumentChunkSyncTaskAppService service = new DocumentChunkSyncTaskAppService(
|
||||||
|
taskMapper,
|
||||||
|
chunkMapper,
|
||||||
|
collectionService,
|
||||||
|
modelService,
|
||||||
|
searcherFactory,
|
||||||
|
producer,
|
||||||
|
transactionManager,
|
||||||
|
redisLockExecutor,
|
||||||
|
Mockito.mock(AiMilvusConfig.class),
|
||||||
|
Mockito.mock(ObjectProvider.class)
|
||||||
|
);
|
||||||
|
return new Fixture(service, taskMapper, chunkMapper, collectionService,
|
||||||
|
producer, task, chunk, taskId, chunkId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private record Fixture(
|
||||||
|
DocumentChunkSyncTaskAppService service,
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper,
|
||||||
|
DocumentChunkMapper chunkMapper,
|
||||||
|
DocumentCollectionService collectionService,
|
||||||
|
DocumentChunkSyncTaskProducer producer,
|
||||||
|
DocumentChunkSyncTask task,
|
||||||
|
DocumentChunk chunk,
|
||||||
|
BigInteger taskId,
|
||||||
|
BigInteger chunkId
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
package tech.easyflow.ai.service.impl;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import org.mockito.InOrder;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.SimpleTransactionStatus;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncState;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncTaskAppService;
|
||||||
|
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.entity.Document;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link DocumentChunkServiceImpl} 异步索引任务回归测试。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkServiceImplTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void updateContentShouldPersistPendingVersionBeforeDispatch() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
|
||||||
|
DocumentChunk result = fixture.service.updateContent(
|
||||||
|
fixture.knowledgeId,
|
||||||
|
fixture.chunkId,
|
||||||
|
"# 新标题\n\n\n正文"
|
||||||
|
);
|
||||||
|
|
||||||
|
ArgumentCaptor<DocumentChunk> update = ArgumentCaptor.forClass(DocumentChunk.class);
|
||||||
|
Mockito.verify(fixture.chunkMapper).update(update.capture());
|
||||||
|
Assert.assertEquals("# 新标题\n\n说明\n正文", update.getValue().getContent());
|
||||||
|
Assert.assertEquals(DocumentChunkSyncState.PENDING, update.getValue().getIndexSyncStatus());
|
||||||
|
Assert.assertEquals(Long.valueOf(3L), update.getValue().getIndexSyncVersion());
|
||||||
|
Assert.assertEquals(
|
||||||
|
"# 新标题\n\n\n正文",
|
||||||
|
update.getValue().getOptions().get(DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN)
|
||||||
|
);
|
||||||
|
Assert.assertEquals(DocumentChunkSyncState.PENDING, result.getIndexSyncStatus());
|
||||||
|
InOrder order = Mockito.inOrder(
|
||||||
|
fixture.syncTaskAppService,
|
||||||
|
fixture.chunkMapper
|
||||||
|
);
|
||||||
|
order.verify(fixture.syncTaskAppService).createTask(
|
||||||
|
Mockito.eq(result),
|
||||||
|
Mockito.eq(fixture.collection),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.OPERATION_UPSERT),
|
||||||
|
Mockito.eq(3L),
|
||||||
|
Mockito.any()
|
||||||
|
);
|
||||||
|
order.verify(fixture.chunkMapper).update(Mockito.any(DocumentChunk.class));
|
||||||
|
Mockito.verify(fixture.syncTaskAppService).dispatchBestEffort(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void updateContentShouldRejectBlankWithoutSideEffects() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
|
||||||
|
try {
|
||||||
|
fixture.service.updateContent(fixture.knowledgeId, fixture.chunkId, " \n ");
|
||||||
|
Assert.fail("空内容必须转删除,不能进入更新链路");
|
||||||
|
} catch (BusinessException expected) {
|
||||||
|
Assert.assertEquals(
|
||||||
|
DocumentChunkServiceImpl.DOCUMENT_CHUNK_EMPTY_REQUIRES_DELETE,
|
||||||
|
expected.getErrorCode()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Mockito.verifyNoInteractions(fixture.chunkMapper, fixture.documentMapper,
|
||||||
|
fixture.syncTaskAppService);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void deleteChunkShouldPersistCleanupTaskAndUpdateParentCount() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
Mockito.when(fixture.chunkMapper.deleteById(fixture.chunkId)).thenReturn(1);
|
||||||
|
Mockito.when(fixture.chunkMapper.selectCountByQuery(Mockito.any(QueryWrapper.class)))
|
||||||
|
.thenReturn(0L);
|
||||||
|
|
||||||
|
DocumentChunkDeleteResult result = fixture.service.deleteChunk(
|
||||||
|
fixture.knowledgeId,
|
||||||
|
fixture.chunkId
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertEquals(0L, result.remainingChunkCount());
|
||||||
|
InOrder order = Mockito.inOrder(
|
||||||
|
fixture.syncTaskAppService,
|
||||||
|
fixture.chunkMapper,
|
||||||
|
fixture.documentMapper
|
||||||
|
);
|
||||||
|
order.verify(fixture.syncTaskAppService).createTask(
|
||||||
|
Mockito.any(), Mockito.eq(fixture.collection),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.OPERATION_DELETE),
|
||||||
|
Mockito.eq(3L), Mockito.any()
|
||||||
|
);
|
||||||
|
order.verify(fixture.chunkMapper).deleteById(fixture.chunkId);
|
||||||
|
order.verify(fixture.documentMapper).update(Mockito.any(Document.class));
|
||||||
|
Mockito.verify(fixture.syncTaskAppService).dispatchBestEffort(fixture.taskId);
|
||||||
|
ArgumentCaptor<Document> parent = ArgumentCaptor.forClass(Document.class);
|
||||||
|
Mockito.verify(fixture.documentMapper).update(parent.capture());
|
||||||
|
Assert.assertEquals(Integer.valueOf(0), parent.getValue().getTotalChunks());
|
||||||
|
Assert.assertEquals(Integer.valueOf(100), parent.getValue().getProgressPercent());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void listStatusShouldRejectChunkFromAnotherKnowledge() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
DocumentChunk foreign = new DocumentChunk();
|
||||||
|
foreign.setId(BigInteger.valueOf(999));
|
||||||
|
foreign.setDocumentId(fixture.documentId);
|
||||||
|
foreign.setDocumentCollectionId(BigInteger.valueOf(888));
|
||||||
|
Mockito.when(fixture.chunkMapper.selectSyncStates(
|
||||||
|
fixture.documentId, List.of(foreign.getId())
|
||||||
|
)).thenReturn(List.of(foreign));
|
||||||
|
|
||||||
|
try {
|
||||||
|
fixture.service.listIndexSyncStatus(
|
||||||
|
fixture.knowledgeId, fixture.documentId, List.of(foreign.getId())
|
||||||
|
);
|
||||||
|
Assert.fail("跨知识库状态查询必须被拒绝");
|
||||||
|
} catch (BusinessException expected) {
|
||||||
|
Assert.assertEquals("分块不存在", expected.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void listStatusShouldResolveKnowledgeFromDocumentForAdminPolling() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
Document parent = new Document();
|
||||||
|
parent.setId(fixture.documentId);
|
||||||
|
parent.setCollectionId(fixture.knowledgeId);
|
||||||
|
Mockito.when(fixture.documentMapper.selectOneById(fixture.documentId))
|
||||||
|
.thenReturn(parent);
|
||||||
|
DocumentChunk state = new DocumentChunk();
|
||||||
|
state.setId(fixture.chunkId);
|
||||||
|
state.setDocumentId(fixture.documentId);
|
||||||
|
state.setDocumentCollectionId(fixture.knowledgeId);
|
||||||
|
state.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
state.setIndexSyncVersion(3L);
|
||||||
|
Mockito.when(fixture.chunkMapper.selectSyncStates(
|
||||||
|
fixture.documentId, List.of(fixture.chunkId)
|
||||||
|
)).thenReturn(List.of(state));
|
||||||
|
|
||||||
|
List<tech.easyflow.ai.dto.DocumentChunkSyncStatus> result =
|
||||||
|
fixture.service.listIndexSyncStatus(
|
||||||
|
null, fixture.documentId, List.of(fixture.chunkId)
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertEquals(1, result.size());
|
||||||
|
Assert.assertEquals(DocumentChunkSyncState.PENDING, result.get(0).indexSyncStatus());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void updateContentShouldExposeRetryableErrorWhenLockIsUnavailable() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
Mockito.when(fixture.redisLockExecutor.tryAcquire(
|
||||||
|
Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
fixture.service.updateContent(fixture.knowledgeId, fixture.chunkId, "更新内容");
|
||||||
|
Assert.fail("锁不可用时必须返回可重试错误");
|
||||||
|
} catch (BusinessException expected) {
|
||||||
|
Assert.assertEquals(503, expected.getHttpStatus());
|
||||||
|
Assert.assertEquals(
|
||||||
|
DocumentChunkServiceImpl.DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
expected.getErrorCode()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Mockito.verify(fixture.chunkMapper, Mockito.never()).update(Mockito.any());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Fixture fixture() {
|
||||||
|
BigInteger knowledgeId = BigInteger.valueOf(101);
|
||||||
|
BigInteger documentId = BigInteger.valueOf(201);
|
||||||
|
BigInteger chunkId = BigInteger.valueOf(301);
|
||||||
|
BigInteger taskId = BigInteger.valueOf(401);
|
||||||
|
|
||||||
|
DocumentChunk current = new DocumentChunk();
|
||||||
|
current.setId(chunkId);
|
||||||
|
current.setDocumentId(documentId);
|
||||||
|
current.setDocumentCollectionId(knowledgeId);
|
||||||
|
current.setContent("旧内容");
|
||||||
|
current.setIndexSyncStatus(DocumentChunkSyncState.SYNCED);
|
||||||
|
current.setIndexSyncVersion(2L);
|
||||||
|
HashMap<String, Object> options = new HashMap<>();
|
||||||
|
options.put("existing", "保留");
|
||||||
|
current.setOptions(options);
|
||||||
|
|
||||||
|
DocumentCollection collection = Mockito.mock(DocumentCollection.class);
|
||||||
|
Mockito.when(collection.getId()).thenReturn(knowledgeId);
|
||||||
|
Mockito.when(collection.getVectorStoreCollection()).thenReturn("kb-test");
|
||||||
|
DocumentCollectionService collectionService = Mockito.mock(DocumentCollectionService.class);
|
||||||
|
Mockito.when(collectionService.getById(knowledgeId)).thenReturn(collection);
|
||||||
|
|
||||||
|
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||||
|
Mockito.when(chunkMapper.selectOneById(chunkId)).thenReturn(current);
|
||||||
|
Mockito.when(chunkMapper.update(Mockito.any(DocumentChunk.class))).thenReturn(1);
|
||||||
|
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||||
|
Mockito.when(documentMapper.update(Mockito.any(Document.class))).thenReturn(1);
|
||||||
|
|
||||||
|
DocumentChunkSyncTask task = new DocumentChunkSyncTask();
|
||||||
|
task.setId(taskId);
|
||||||
|
DocumentChunkSyncTaskAppService syncTaskAppService =
|
||||||
|
Mockito.mock(DocumentChunkSyncTaskAppService.class);
|
||||||
|
Mockito.when(syncTaskAppService.createTask(
|
||||||
|
Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyLong(), Mockito.any()
|
||||||
|
)).thenReturn(task);
|
||||||
|
|
||||||
|
PlatformTransactionManager transactionManager = Mockito.mock(PlatformTransactionManager.class);
|
||||||
|
Mockito.when(transactionManager.getTransaction(Mockito.any()))
|
||||||
|
.thenReturn(new SimpleTransactionStatus());
|
||||||
|
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||||
|
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
|
||||||
|
Mockito.when(redisLockExecutor.tryAcquire(
|
||||||
|
Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(lockHandle);
|
||||||
|
|
||||||
|
DocumentChunkServiceImpl service = new DocumentChunkServiceImpl(
|
||||||
|
chunkMapper,
|
||||||
|
documentMapper,
|
||||||
|
collectionService,
|
||||||
|
syncTaskAppService,
|
||||||
|
transactionManager,
|
||||||
|
redisLockExecutor
|
||||||
|
);
|
||||||
|
return new Fixture(service, knowledgeId, documentId, chunkId, taskId,
|
||||||
|
chunkMapper, documentMapper, collection, syncTaskAppService,
|
||||||
|
redisLockExecutor);
|
||||||
|
}
|
||||||
|
|
||||||
|
private record Fixture(
|
||||||
|
DocumentChunkServiceImpl service,
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger documentId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
BigInteger taskId,
|
||||||
|
DocumentChunkMapper chunkMapper,
|
||||||
|
DocumentMapper documentMapper,
|
||||||
|
DocumentCollection collection,
|
||||||
|
DocumentChunkSyncTaskAppService syncTaskAppService,
|
||||||
|
RedisLockExecutor redisLockExecutor
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ package tech.easyflow.ai.service.impl;
|
|||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
import com.easyagents.search.engine.service.KeywordSearchRequest;
|
import com.easyagents.search.engine.service.KeywordSearchRequest;
|
||||||
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
@@ -20,6 +21,7 @@ import java.math.BigInteger;
|
|||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_DOC_RECALL_MAX_NUM;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_DOC_RECALL_MAX_NUM;
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SIMILARITY_THRESHOLD;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SIMILARITY_THRESHOLD;
|
||||||
@@ -100,7 +102,9 @@ public class DocumentCollectionServiceImplTest {
|
|||||||
|
|
||||||
DocumentCollectionServiceImpl service = new TestDocumentCollectionService(collection);
|
DocumentCollectionServiceImpl service = new TestDocumentCollectionService(collection);
|
||||||
setField(service, "searcherFactory", new SearcherFactory(new StaticObjectProvider<DocumentSearcher>(searcher)));
|
setField(service, "searcherFactory", new SearcherFactory(new StaticObjectProvider<DocumentSearcher>(searcher)));
|
||||||
setField(service, "documentChunkMapper", mockDocumentChunkMapper(completedChunk, indexingChunk));
|
AtomicReference<QueryWrapper> chunkQuery = new AtomicReference<QueryWrapper>();
|
||||||
|
setField(service, "documentChunkMapper",
|
||||||
|
mockDocumentChunkMapper(chunkQuery, completedChunk, indexingChunk));
|
||||||
setField(service, "documentMapper", mockDocumentMapper(completedDocument));
|
setField(service, "documentMapper", mockDocumentMapper(completedDocument));
|
||||||
|
|
||||||
tech.easyflow.ai.rag.KnowledgeRetrievalRequest request = new tech.easyflow.ai.rag.KnowledgeRetrievalRequest();
|
tech.easyflow.ai.rag.KnowledgeRetrievalRequest request = new tech.easyflow.ai.rag.KnowledgeRetrievalRequest();
|
||||||
@@ -116,6 +120,10 @@ public class DocumentCollectionServiceImplTest {
|
|||||||
Assert.assertEquals(completedDocument.getTitle(), result.get(0).getTitle());
|
Assert.assertEquals(completedDocument.getTitle(), result.get(0).getTitle());
|
||||||
Assert.assertEquals("completed chunk", result.get(0).getContent());
|
Assert.assertEquals("completed chunk", result.get(0).getContent());
|
||||||
Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId);
|
Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId);
|
||||||
|
Assert.assertTrue(
|
||||||
|
"检索补齐必须过滤尚未同步的分块",
|
||||||
|
chunkQuery.get().toSQL().toUpperCase().contains("INDEX_SYNC_STATUS")
|
||||||
|
);
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
tech.easyflow.ai.entity.DocumentCollection.TYPE_DOCUMENT,
|
tech.easyflow.ai.entity.DocumentCollection.TYPE_DOCUMENT,
|
||||||
result.get(0).getMetadata("resultType")
|
result.get(0).getMetadata("resultType")
|
||||||
@@ -210,7 +218,10 @@ public class DocumentCollectionServiceImplTest {
|
|||||||
return document;
|
return document;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static DocumentChunkMapper mockDocumentChunkMapper(tech.easyflow.ai.entity.DocumentChunk... chunks) {
|
private static DocumentChunkMapper mockDocumentChunkMapper(
|
||||||
|
AtomicReference<QueryWrapper> query,
|
||||||
|
tech.easyflow.ai.entity.DocumentChunk... chunks
|
||||||
|
) {
|
||||||
Map<String, tech.easyflow.ai.entity.DocumentChunk> chunkMap = new HashMap<String, tech.easyflow.ai.entity.DocumentChunk>();
|
Map<String, tech.easyflow.ai.entity.DocumentChunk> chunkMap = new HashMap<String, tech.easyflow.ai.entity.DocumentChunk>();
|
||||||
for (tech.easyflow.ai.entity.DocumentChunk chunk : chunks) {
|
for (tech.easyflow.ai.entity.DocumentChunk chunk : chunks) {
|
||||||
chunkMap.put(String.valueOf(chunk.getId()), chunk);
|
chunkMap.put(String.valueOf(chunk.getId()), chunk);
|
||||||
@@ -220,6 +231,7 @@ public class DocumentCollectionServiceImplTest {
|
|||||||
new Class<?>[]{DocumentChunkMapper.class},
|
new Class<?>[]{DocumentChunkMapper.class},
|
||||||
(proxy, method, args) -> {
|
(proxy, method, args) -> {
|
||||||
if ("selectListByQuery".equals(method.getName())) {
|
if ("selectListByQuery".equals(method.getName())) {
|
||||||
|
query.set((QueryWrapper) args[0]);
|
||||||
return List.copyOf(chunkMap.values());
|
return List.copyOf(chunkMap.values());
|
||||||
}
|
}
|
||||||
return defaultValue(method.getReturnType());
|
return defaultValue(method.getReturnType());
|
||||||
|
|||||||
@@ -3,22 +3,27 @@ package tech.easyflow.ai.service.impl;
|
|||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.core.store.StoreResult;
|
import com.easyagents.core.store.StoreResult;
|
||||||
import com.easyagents.rag.core.RagDefaults;
|
import com.easyagents.rag.core.RagDefaults;
|
||||||
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.mockito.ArgumentCaptor;
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import org.mockito.InOrder;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncState;
|
||||||
import tech.easyflow.ai.entity.Document;
|
import tech.easyflow.ai.entity.Document;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.Model;
|
import tech.easyflow.ai.entity.Model;
|
||||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
@@ -28,6 +33,7 @@ import java.math.BigInteger;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link DocumentServiceImpl} 文档维护回归测试。
|
* {@link DocumentServiceImpl} 文档维护回归测试。
|
||||||
@@ -155,9 +161,15 @@ public class DocumentServiceImplTest {
|
|||||||
setField(service, "modelService", modelService);
|
setField(service, "modelService", modelService);
|
||||||
setField(service, "storageService", storageService);
|
setField(service, "storageService", storageService);
|
||||||
setField(service, "searcherFactory", searcherFactory);
|
setField(service, "searcherFactory", searcherFactory);
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper = configureRemovalCoordination(service);
|
||||||
|
|
||||||
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
||||||
|
|
||||||
|
InOrder order = Mockito.inOrder(taskMapper, documentStore);
|
||||||
|
order.verify(taskMapper).supersedeChunk(
|
||||||
|
Mockito.eq(BigInteger.valueOf(201)), Mockito.any()
|
||||||
|
);
|
||||||
|
order.verify(documentStore).delete(Mockito.anyList(), Mockito.any());
|
||||||
Mockito.verify(documentMapper).deleteById(documentId);
|
Mockito.verify(documentMapper).deleteById(documentId);
|
||||||
Mockito.verify(storageService).delete(document.getDocumentPath());
|
Mockito.verify(storageService).delete(document.getDocumentPath());
|
||||||
}
|
}
|
||||||
@@ -206,6 +218,7 @@ public class DocumentServiceImplTest {
|
|||||||
setField(service, "modelService", modelService);
|
setField(service, "modelService", modelService);
|
||||||
setField(service, "storageService", storageService);
|
setField(service, "storageService", storageService);
|
||||||
setField(service, "searcherFactory", searcherFactory);
|
setField(service, "searcherFactory", searcherFactory);
|
||||||
|
configureRemovalCoordination(service);
|
||||||
|
|
||||||
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
||||||
|
|
||||||
@@ -238,6 +251,7 @@ public class DocumentServiceImplTest {
|
|||||||
setField(service, "documentMapper", documentMapper);
|
setField(service, "documentMapper", documentMapper);
|
||||||
setField(service, "documentChunkMapper", chunkMapper);
|
setField(service, "documentChunkMapper", chunkMapper);
|
||||||
setField(service, "knowledgeService", knowledgeService);
|
setField(service, "knowledgeService", knowledgeService);
|
||||||
|
configureRemovalCoordination(service);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
service.removeDoc(documentId.toString());
|
service.removeDoc(documentId.toString());
|
||||||
@@ -295,6 +309,7 @@ public class DocumentServiceImplTest {
|
|||||||
setField(service, "modelService", modelService);
|
setField(service, "modelService", modelService);
|
||||||
setField(service, "storageService", storageService);
|
setField(service, "storageService", storageService);
|
||||||
setField(service, "searcherFactory", searcherFactory);
|
setField(service, "searcherFactory", searcherFactory);
|
||||||
|
configureRemovalCoordination(service);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
service.removeDoc(documentId.toString());
|
service.removeDoc(documentId.toString());
|
||||||
@@ -308,6 +323,176 @@ public class DocumentServiceImplTest {
|
|||||||
Mockito.verifyNoInteractions(storageService);
|
Mockito.verifyNoInteractions(storageService);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证关键词索引拒绝删除时保留数据库记录,供用户安全重试。
|
||||||
|
*
|
||||||
|
* @throws Exception 反射注入异常
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void removeDocShouldStopWhenKeywordDeleteFails() throws Exception {
|
||||||
|
BigInteger documentId = BigInteger.valueOf(451);
|
||||||
|
BigInteger knowledgeId = BigInteger.valueOf(452);
|
||||||
|
BigInteger modelId = BigInteger.valueOf(453);
|
||||||
|
BigInteger chunkId = BigInteger.valueOf(454);
|
||||||
|
Document document = new Document();
|
||||||
|
document.setId(documentId);
|
||||||
|
document.setCollectionId(knowledgeId);
|
||||||
|
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||||
|
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||||
|
Mockito.when(documentStore.delete(Mockito.anyList(), Mockito.any()))
|
||||||
|
.thenReturn(StoreResult.success());
|
||||||
|
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
|
||||||
|
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
|
||||||
|
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
|
||||||
|
Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore);
|
||||||
|
Model model = new Model();
|
||||||
|
model.setModelName("embedding-test");
|
||||||
|
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||||
|
Mockito.when(documentMapper.selectOneByQuery(Mockito.any()))
|
||||||
|
.thenReturn(document);
|
||||||
|
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||||
|
Mockito.when(chunkMapper.selectListByQueryAs(
|
||||||
|
Mockito.any(), Mockito.eq(BigInteger.class)))
|
||||||
|
.thenReturn(List.of(chunkId));
|
||||||
|
DocumentCollectionService knowledgeService = Mockito.mock(
|
||||||
|
DocumentCollectionService.class
|
||||||
|
);
|
||||||
|
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
|
||||||
|
ModelService modelService = Mockito.mock(ModelService.class);
|
||||||
|
Mockito.when(modelService.getById(modelId)).thenReturn(model);
|
||||||
|
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||||
|
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
|
||||||
|
Mockito.when(searcher.deleteDocument(chunkId)).thenReturn(false);
|
||||||
|
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
|
||||||
|
Mockito.when(searcherFactory.getSearcher()).thenReturn(searcher);
|
||||||
|
|
||||||
|
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||||
|
setField(service, "documentMapper", documentMapper);
|
||||||
|
setField(service, "documentChunkMapper", chunkMapper);
|
||||||
|
setField(service, "knowledgeService", knowledgeService);
|
||||||
|
setField(service, "modelService", modelService);
|
||||||
|
setField(service, "storageService", storageService);
|
||||||
|
setField(service, "searcherFactory", searcherFactory);
|
||||||
|
configureRemovalCoordination(service);
|
||||||
|
|
||||||
|
try {
|
||||||
|
service.removeDoc(documentId.toString());
|
||||||
|
Assert.fail("关键词索引删除失败时应停止文档删除");
|
||||||
|
} catch (BusinessException expected) {
|
||||||
|
Assert.assertEquals("文档关键词索引删除失败", expected.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
Mockito.verify(chunkMapper, Mockito.never()).deleteByQuery(Mockito.any());
|
||||||
|
Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any());
|
||||||
|
Mockito.verifyNoInteractions(storageService);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证外部与数据库删除已经执行完成时,父锁收尾阶段丢失不会回滚数据库事务。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void removeDocShouldCommitAfterPostExecutionLockLoss() throws Exception {
|
||||||
|
BigInteger documentId = BigInteger.valueOf(501);
|
||||||
|
BigInteger knowledgeId = BigInteger.valueOf(502);
|
||||||
|
Document document = new Document();
|
||||||
|
document.setId(documentId);
|
||||||
|
document.setCollectionId(knowledgeId);
|
||||||
|
document.setDocumentPath("storage://lock-lost.txt");
|
||||||
|
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||||
|
|
||||||
|
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||||
|
Mockito.when(documentMapper.selectOneByQuery(Mockito.any()))
|
||||||
|
.thenReturn(document);
|
||||||
|
Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1);
|
||||||
|
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||||
|
Mockito.when(chunkMapper.selectListByQueryAs(
|
||||||
|
Mockito.any(), Mockito.eq(BigInteger.class))).thenReturn(List.of());
|
||||||
|
Mockito.when(chunkMapper.deleteByQuery(Mockito.any())).thenReturn(0);
|
||||||
|
DocumentCollectionService knowledgeService = Mockito.mock(
|
||||||
|
DocumentCollectionService.class
|
||||||
|
);
|
||||||
|
Mockito.when(knowledgeService.getById(knowledgeId))
|
||||||
|
.thenReturn(Mockito.mock(DocumentCollection.class));
|
||||||
|
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||||
|
|
||||||
|
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||||
|
setField(service, "documentMapper", documentMapper);
|
||||||
|
setField(service, "documentChunkMapper", chunkMapper);
|
||||||
|
setField(service, "knowledgeService", knowledgeService);
|
||||||
|
setField(service, "storageService", storageService);
|
||||||
|
setField(service, "searcherFactory", Mockito.mock(SearcherFactory.class));
|
||||||
|
configureRemovalCoordination(service);
|
||||||
|
RedisLockExecutor lockExecutor = getField(
|
||||||
|
service, "redisLockExecutor", RedisLockExecutor.class
|
||||||
|
);
|
||||||
|
Mockito.doAnswer(invocation -> {
|
||||||
|
String lockKey = invocation.getArgument(0);
|
||||||
|
Object result = invocation.<Supplier<?>>getArgument(3).get();
|
||||||
|
if (DocumentChunkSyncState.parentLockKey(documentId.toString())
|
||||||
|
.equals(lockKey)) {
|
||||||
|
throw new IllegalStateException("模拟 callback 完成后续租丢失");
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}).when(lockExecutor).executeWithRenewingLock(
|
||||||
|
Mockito.anyString(), Mockito.any(), Mockito.any(),
|
||||||
|
Mockito.<Supplier<?>>any()
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
||||||
|
Mockito.verify(documentMapper).deleteById(documentId);
|
||||||
|
Mockito.verify(storageService).delete(document.getDocumentPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证向量存储配置缺失时不会先废弃仍可恢复的分块同步任务。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void removeDocShouldKeepSyncTasksWhenVectorStoreIsUnavailable()
|
||||||
|
throws Exception {
|
||||||
|
BigInteger documentId = BigInteger.valueOf(601);
|
||||||
|
BigInteger knowledgeId = BigInteger.valueOf(602);
|
||||||
|
BigInteger chunkId = BigInteger.valueOf(603);
|
||||||
|
Document document = new Document();
|
||||||
|
document.setId(documentId);
|
||||||
|
document.setCollectionId(knowledgeId);
|
||||||
|
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||||
|
|
||||||
|
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
|
||||||
|
Mockito.when(knowledge.toDocumentStore()).thenReturn(null);
|
||||||
|
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||||
|
Mockito.when(documentMapper.selectOneByQuery(Mockito.any()))
|
||||||
|
.thenReturn(document);
|
||||||
|
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||||
|
Mockito.when(chunkMapper.selectListByQueryAs(
|
||||||
|
Mockito.any(), Mockito.eq(BigInteger.class)))
|
||||||
|
.thenReturn(List.of(chunkId));
|
||||||
|
DocumentCollectionService knowledgeService = Mockito.mock(
|
||||||
|
DocumentCollectionService.class
|
||||||
|
);
|
||||||
|
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
|
||||||
|
|
||||||
|
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||||
|
setField(service, "documentMapper", documentMapper);
|
||||||
|
setField(service, "documentChunkMapper", chunkMapper);
|
||||||
|
setField(service, "knowledgeService", knowledgeService);
|
||||||
|
setField(service, "modelService", Mockito.mock(ModelService.class));
|
||||||
|
setField(service, "storageService", Mockito.mock(FileStorageService.class));
|
||||||
|
setField(service, "searcherFactory", Mockito.mock(SearcherFactory.class));
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper = configureRemovalCoordination(service);
|
||||||
|
|
||||||
|
try {
|
||||||
|
service.removeDoc(documentId.toString());
|
||||||
|
Assert.fail("向量存储不可用时应停止文档删除");
|
||||||
|
} catch (BusinessException expected) {
|
||||||
|
Assert.assertEquals("文档向量存储不可用", expected.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
Mockito.verify(taskMapper, Mockito.never())
|
||||||
|
.supersedeChunk(Mockito.any(), Mockito.any());
|
||||||
|
Mockito.verify(chunkMapper, Mockito.never()).deleteByQuery(Mockito.any());
|
||||||
|
Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证旧版向量化入口同样拒绝超过 BGE-M3 上下文预算的分块。
|
* 验证旧版向量化入口同样拒绝超过 BGE-M3 上下文预算的分块。
|
||||||
*
|
*
|
||||||
@@ -351,6 +536,33 @@ public class DocumentServiceImplTest {
|
|||||||
field.set(target, value);
|
field.set(target, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static <T> T getField(Object target,
|
||||||
|
String fieldName,
|
||||||
|
Class<T> fieldType) throws Exception {
|
||||||
|
Field field = DocumentServiceImpl.class.getDeclaredField(fieldName);
|
||||||
|
field.setAccessible(true);
|
||||||
|
return fieldType.cast(field.get(target));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DocumentChunkSyncTaskMapper configureRemovalCoordination(
|
||||||
|
DocumentServiceImpl service
|
||||||
|
) throws Exception {
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper = Mockito.mock(
|
||||||
|
DocumentChunkSyncTaskMapper.class
|
||||||
|
);
|
||||||
|
RedisLockExecutor lockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||||
|
Mockito.doAnswer(invocation -> invocation.<Supplier<?>>getArgument(3).get())
|
||||||
|
.when(lockExecutor).executeWithRenewingLock(
|
||||||
|
Mockito.anyString(),
|
||||||
|
Mockito.any(),
|
||||||
|
Mockito.any(),
|
||||||
|
Mockito.<Supplier<?>>any()
|
||||||
|
);
|
||||||
|
setField(service, "documentChunkSyncTaskMapper", taskMapper);
|
||||||
|
setField(service, "redisLockExecutor", lockExecutor);
|
||||||
|
return taskMapper;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 统一 SQL 文本格式,便于断言查询结构。
|
* 统一 SQL 文本格式,便于断言查询结构。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -353,6 +353,13 @@ rag:
|
|||||||
username: easyflowadmin
|
username: easyflowadmin
|
||||||
password: easyflowadmin123
|
password: easyflowadmin123
|
||||||
autoCreateCollection: true
|
autoCreateCollection: true
|
||||||
|
poolMaxTotal: 8
|
||||||
|
poolMaxTotalPerKey: 8
|
||||||
|
poolMaxIdlePerKey: 4
|
||||||
|
poolMinIdlePerKey: 1
|
||||||
|
poolMaxWaitMillis: 3000
|
||||||
|
poolEvictionIntervalMillis: 60000
|
||||||
|
poolMinEvictableIdleMillis: 300000
|
||||||
# 搜索引擎配置
|
# 搜索引擎配置
|
||||||
searcher:
|
searcher:
|
||||||
lucene:
|
lucene:
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
ALTER TABLE `tb_document_chunk`
|
||||||
|
ADD COLUMN `index_sync_status` varchar(16) NOT NULL DEFAULT 'SYNCED'
|
||||||
|
COMMENT '检索索引同步状态' AFTER `options`,
|
||||||
|
ADD COLUMN `index_sync_version` bigint UNSIGNED NOT NULL DEFAULT 0
|
||||||
|
COMMENT '检索索引同步版本' AFTER `index_sync_status`,
|
||||||
|
ADD COLUMN `index_sync_error_code` varchar(64) NULL DEFAULT NULL
|
||||||
|
COMMENT '脱敏同步错误码' AFTER `index_sync_version`,
|
||||||
|
ADD COLUMN `index_sync_error_message` varchar(255) NULL DEFAULT NULL
|
||||||
|
COMMENT '脱敏同步错误摘要' AFTER `index_sync_error_code`,
|
||||||
|
ADD KEY `idx_document_chunk_sync_filter`
|
||||||
|
(`document_collection_id`, `index_sync_status`, `id`);
|
||||||
|
|
||||||
|
CREATE TABLE `tb_document_chunk_sync_task`
|
||||||
|
(
|
||||||
|
`id` bigint UNSIGNED NOT NULL COMMENT '主键',
|
||||||
|
`chunk_id` bigint UNSIGNED NOT NULL COMMENT '分块ID',
|
||||||
|
`document_id` bigint UNSIGNED NOT NULL COMMENT '文档ID',
|
||||||
|
`document_collection_id` bigint UNSIGNED NOT NULL COMMENT '知识库ID',
|
||||||
|
`vector_collection` varchar(128) NULL DEFAULT NULL COMMENT '向量集合名',
|
||||||
|
`operation` varchar(16) NOT NULL COMMENT 'UPSERT/DELETE',
|
||||||
|
`sync_version` bigint UNSIGNED NOT NULL COMMENT '同步版本',
|
||||||
|
`status` varchar(16) NOT NULL DEFAULT 'PENDING'
|
||||||
|
COMMENT 'PENDING/RUNNING/SUCCEEDED/FAILED/SUPERSEDED',
|
||||||
|
`attempt_count` int NOT NULL DEFAULT 0 COMMENT '已尝试次数',
|
||||||
|
`next_retry_at` datetime(3) NOT NULL COMMENT '下次执行时间',
|
||||||
|
`last_dispatched_at` datetime(3) NULL DEFAULT NULL COMMENT '最近投递时间',
|
||||||
|
`execution_token` varchar(64) NULL DEFAULT NULL COMMENT '执行令牌',
|
||||||
|
`lease_until` datetime(3) NULL DEFAULT NULL COMMENT '租约截止时间',
|
||||||
|
`error_code` varchar(64) NULL DEFAULT NULL COMMENT '脱敏错误码',
|
||||||
|
`error_message` varchar(255) NULL DEFAULT NULL COMMENT '脱敏错误摘要',
|
||||||
|
`created` datetime(3) NULL DEFAULT NULL COMMENT '创建时间',
|
||||||
|
`created_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '创建人',
|
||||||
|
`modified` datetime(3) NULL DEFAULT NULL COMMENT '修改时间',
|
||||||
|
`modified_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '修改人',
|
||||||
|
PRIMARY KEY (`id`) USING BTREE,
|
||||||
|
UNIQUE KEY `uk_document_chunk_sync_version`
|
||||||
|
(`chunk_id`, `sync_version`, `operation`),
|
||||||
|
KEY `idx_document_chunk_sync_due`
|
||||||
|
(`status`, `next_retry_at`, `last_dispatched_at`, `id`),
|
||||||
|
KEY `idx_document_chunk_sync_lease`
|
||||||
|
(`status`, `lease_until`, `id`),
|
||||||
|
KEY `idx_document_chunk_sync_chunk`
|
||||||
|
(`chunk_id`, `sync_version`, `status`)
|
||||||
|
) ENGINE = InnoDB
|
||||||
|
CHARACTER SET = utf8mb4
|
||||||
|
COLLATE = utf8mb4_0900_ai_ci
|
||||||
|
COMMENT = '文档分块检索索引同步任务'
|
||||||
|
ROW_FORMAT = DYNAMIC;
|
||||||
6
pom.xml
6
pom.xml
@@ -52,6 +52,7 @@
|
|||||||
<proguard.version>7.9.1</proguard.version>
|
<proguard.version>7.9.1</proguard.version>
|
||||||
<proguard.maven.plugin.version>2.7.0</proguard.maven.plugin.version>
|
<proguard.maven.plugin.version>2.7.0</proguard.maven.plugin.version>
|
||||||
<re2j.version>1.8</re2j.version>
|
<re2j.version>1.8</re2j.version>
|
||||||
|
<milvus.version>2.3.11</milvus.version>
|
||||||
</properties>
|
</properties>
|
||||||
<dependencyManagement>
|
<dependencyManagement>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
@@ -184,6 +185,11 @@
|
|||||||
<artifactId>easy-agents-bom</artifactId>
|
<artifactId>easy-agents-bom</artifactId>
|
||||||
<version>${easy-agents.version}</version>
|
<version>${easy-agents.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.milvus</groupId>
|
||||||
|
<artifactId>milvus-sdk-java</artifactId>
|
||||||
|
<version>${milvus.version}</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.easyagents</groupId>
|
<groupId>com.easyagents</groupId>
|
||||||
<artifactId>easy-agents-flow</artifactId>
|
<artifactId>easy-agents-flow</artifactId>
|
||||||
|
|||||||
Reference in New Issue
Block a user