EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

发布 v1.10 #5

Merged
czm merged 147 commits from develop into main 2026-08-20 11:36:27 +08:00
Conversation 0 Commits 147 Files Changed 1488 +129 -17
3 changed files with 129 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 742a4b1647 - Show all commits

52
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java
View File

@@ -22,7 +22,6 @@ import com.easyagents.search.engine.service.DocumentSearcher;
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryMethods;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.slf4j.Logger;
@@ -110,7 +109,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
@Override
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null);
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null, false);
}
/**
@@ -129,7 +128,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
int pageNum,
BigInteger documentId
) {
return queryDocumentList(knowledgeId, pageSize, pageNum, null, documentId);
return queryDocumentList(knowledgeId, pageSize, pageNum, null, documentId, true);
}
/**
@@ -140,6 +139,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
* @param pageNum 页码
* @param fileName 可选的文件标题筛选
* @param documentId 可选的文档 ID
* @param includeDetailFields 是否返回正文、存储路径和扩展配置等详情字段
* @return 文档分页
*/
private Page<Document> queryDocumentList(
@@ -147,27 +147,45 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
int pageSize,
int pageNum,
String fileName,
BigInteger documentId
BigInteger documentId,
boolean includeDetailFields
) {
QueryWrapper queryWrapper=QueryWrapper.create()
.select(
DOCUMENT.ALL_COLUMNS,
QueryMethods.count(DOCUMENT_CHUNK.DOCUMENT_ID).as("chunk_count")
)
.from(Document.class)
.leftJoin(DocumentChunk.class).on(DOCUMENT.ID.eq(DOCUMENT_CHUNK.DOCUMENT_ID))
.where(DOCUMENT.COLLECTION_ID.eq(knowledgeId))
.orderBy(DOCUMENT.ID, false)
;
QueryWrapper queryWrapper = QueryWrapper.create().from(Document.class);
if (includeDetailFields) {
// 公开文档分页继续保留历史字段契约，但分块数直接使用文档表中的持久化统计。
queryWrapper.select(
DOCUMENT.ALL_COLUMNS,
DOCUMENT.TOTAL_CHUNKS.as("chunk_count")
);
} else {
// 列表页只读取展示字段，避免批量导入期间反复传输 LONGTEXT 正文和扩展配置。
queryWrapper.select(
DOCUMENT.ID,
DOCUMENT.COLLECTION_ID,
DOCUMENT.DOCUMENT_TYPE,
DOCUMENT.TITLE,
DOCUMENT.CONTENT_TYPE,
DOCUMENT.PROCESS_STATUS,
DOCUMENT.TOTAL_CHUNKS,
DOCUMENT.COMPLETED_CHUNKS,
DOCUMENT.FAILED_CHUNKS,
DOCUMENT.PROGRESS_PERCENT,
DOCUMENT.LAST_TASK_ERROR,
DOCUMENT.TASK_MODIFIED_AT,
DOCUMENT.CREATED,
DOCUMENT.MODIFIED,
DOCUMENT.TOTAL_CHUNKS.as("chunk_count")
);
}
queryWrapper
.where(DOCUMENT.COLLECTION_ID.eq(knowledgeId))
.orderBy(DOCUMENT.ID, false);
if (fileName != null && !fileName.trim().isEmpty()) {
queryWrapper.and(DOCUMENT.TITLE.like(fileName));
}
if (documentId != null) {
queryWrapper.and(DOCUMENT.ID.eq(documentId));
}
// 分组
queryWrapper.groupBy(DOCUMENT.ID);
return documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class);
}

90
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentServiceImplTest.java
View File

@@ -5,6 +5,7 @@ import com.easyagents.core.store.StoreResult;
import com.mybatisflex.core.query.QueryWrapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import tech.easyflow.ai.config.SearcherFactory;
import tech.easyflow.ai.entity.Document;
@@ -21,6 +22,8 @@ import tech.easyflow.common.web.exceptions.BusinessException;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;
/**
* {@link DocumentServiceImpl} 文档维护回归测试。
@@ -30,6 +33,70 @@ import java.util.List;
*/
public class DocumentServiceImplTest {
/**
* 验证管理端列表只查询展示字段，并直接使用文档表中的分块统计。
*
* @throws Exception 反射注入异常
*/
@Test
public void getDocumentListShouldAvoidChunkJoinAndLargeFields() throws Exception {
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
DocumentServiceImpl service = new DocumentServiceImpl();
setField(service, "documentMapper", documentMapper);
service.getDocumentList("1001", 10, 1, null);
ArgumentCaptor<QueryWrapper> queryCaptor =
ArgumentCaptor.forClass(QueryWrapper.class);
Mockito.verify(documentMapper).paginateAs(
Mockito.eq(1),
Mockito.eq(10),
queryCaptor.capture(),
Mockito.eq(Document.class)
);
String sql = normalizeSql(queryCaptor.getValue().toSQL());
Assert.assertFalse(sql.contains("tb_document_chunk"));
Assert.assertFalse(containsSqlIdentifier(sql, "content"));
Assert.assertFalse(containsSqlIdentifier(sql, "options"));
Assert.assertFalse(sql.contains("group by"));
Assert.assertTrue(containsSqlIdentifier(sql, "total_chunks"));
Assert.assertTrue(containsSqlIdentifier(sql, "chunk_count"));
}
/**
* 验证公开文档分页继续返回历史详情字段，同时不再关联分块表聚合。
*
* @throws Exception 反射注入异常
*/
@Test
public void getDocumentListByIdShouldKeepDetailFieldsWithoutChunkJoin()
throws Exception {
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
DocumentServiceImpl service = new DocumentServiceImpl();
setField(service, "documentMapper", documentMapper);
service.getDocumentListById(
"1001",
10,
1,
BigInteger.valueOf(2002)
);
ArgumentCaptor<QueryWrapper> queryCaptor =
ArgumentCaptor.forClass(QueryWrapper.class);
Mockito.verify(documentMapper).paginateAs(
Mockito.eq(1),
Mockito.eq(10),
queryCaptor.capture(),
Mockito.eq(Document.class)
);
String sql = normalizeSql(queryCaptor.getValue().toSQL());
Assert.assertFalse(sql.contains("tb_document_chunk"));
Assert.assertFalse(sql.contains("group by"));
Assert.assertTrue(sql.contains("*"));
Assert.assertTrue(containsSqlIdentifier(sql, "chunk_count"));
}
/**
* 验证删除链路在外部索引和分块清理后删除文档主记录。
*
@@ -252,4 +319,27 @@ public class DocumentServiceImplTest {
field.setAccessible(true);
field.set(target, value);
}
/**
* 统一 SQL 文本格式，便于断言查询结构。
*
* @param sql 原始 SQL
* @return 去除标识符引号并转为小写的 SQL
*/
private static String normalizeSql(String sql) {
return sql.replace("`", "").toLowerCase(Locale.ROOT);
}
/**
* 判断 SQL 是否包含完整列标识符，避免与同前缀列名混淆。
*
* @param sql 已标准化的 SQL
* @param identifier 列标识符
* @return 包含完整标识符时返回 true
*/
private static boolean containsSqlIdentifier(String sql, String identifier) {
return Pattern.compile("\\b" + Pattern.quote(identifier) + "\\b")
.matcher(sql)
.find();
}
}

4
easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V50__mysql_document_chunk_document_index.sql Normal file
View File

@@ -0,0 +1,4 @@
ALTER TABLE `tb_document_chunk`
ADD INDEX `idx_document_chunk_document_id` (`document_id`) USING BTREE,
ALGORITHM=INPLACE,
LOCK=NONE;