@@ -1,5 +1,13 @@
package tech.easyflow.ai.service.impl ;
import cn.idev.excel.EasyExcel ;
import cn.idev.excel.ExcelWriter ;
import cn.idev.excel.FastExcel ;
import cn.idev.excel.context.AnalysisContext ;
import cn.idev.excel.metadata.data.ReadCellData ;
import cn.idev.excel.read.listener.ReadListener ;
import cn.idev.excel.write.metadata.WriteSheet ;
import cn.idev.excel.write.style.column.SimpleColumnWidthStyleStrategy ;
import cn.dev33.satoken.stp.StpUtil ;
import com.easyagents.core.model.embedding.EmbeddingModel ;
import com.easyagents.core.model.embedding.EmbeddingOptions ;
@@ -18,6 +26,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.stereotype.Service ;
import org.springframework.transaction.annotation.Transactional ;
import org.springframework.transaction.PlatformTransactionManager ;
import org.springframework.transaction.TransactionDefinition ;
import org.springframework.transaction.support.TransactionTemplate ;
import org.springframework.web.multipart.MultipartFile ;
import tech.easyflow.ai.config.SearcherFactory ;
import tech.easyflow.ai.entity.DocumentCollection ;
import tech.easyflow.ai.entity.FaqItem ;
@@ -27,12 +39,17 @@ import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.FaqCategoryService ;
import tech.easyflow.ai.service.FaqItemService ;
import tech.easyflow.ai.service.ModelService ;
import tech.easyflow.ai.vo.FaqImportErrorRowVo ;
import tech.easyflow.ai.vo.FaqImportResultVo ;
import tech.easyflow.common.util.StringUtil ;
import tech.easyflow.common.web.exceptions.BusinessException ;
import javax.annotation.Resource ;
import java.io.InputStream ;
import java.io.OutputStream ;
import java.math.BigInteger ;
import java.util.* ;
import java.util.function.Supplier ;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL ;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE ;
@@ -49,6 +66,15 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
private static final String IMAGE_FIELD_URL = " url " ;
private static final String IMAGE_FIELD_ALT = " alt " ;
private static final String IMAGE_FIELD_TITLE = " title " ;
private static final String IMPORT_HEAD_CATEGORY_PATH = " 分类路径 " ;
private static final String IMPORT_HEAD_QUESTION = " 问题 " ;
private static final String IMPORT_HEAD_ANSWER_HTML = " 答案HTML " ;
private static final String DEFAULT_CATEGORY_NAME = " 默认分类 " ;
private static final String IMPORT_UPSERT_KEY_SEPARATOR = " \ u0001 " ;
private static final int MAX_IMPORT_ROWS = 5000 ;
private static final long MAX_IMPORT_FILE_SIZE_BYTES = 10L * 1024L * 1024L ;
private static final int TEMPLATE_COLUMN_WIDTH = 30 ;
private static final int TEMPLATE_NOTE_COLUMN_WIDTH = 90 ;
@Resource
private DocumentCollectionService documentCollectionService ;
@@ -62,6 +88,9 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
@Autowired
private SearcherFactory searcherFactory ;
@Resource
private PlatformTransactionManager transactionManager ;
@Override
@Transactional
public boolean saveFaqItem ( FaqItem entity ) {
@@ -136,6 +165,133 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
return removeById ( id ) ;
}
@Override
public FaqImportResultVo importFromExcel ( BigInteger collectionId , MultipartFile file ) {
validateImportFile ( file ) ;
getFaqCollection ( collectionId ) ;
BigInteger defaultCategoryId = faqCategoryService . ensureDefaultCategory ( collectionId ) ;
Map < BigInteger , String > categoryPathMap = faqCategoryService . buildPathMap ( collectionId ) ;
String defaultCategoryPath = normalizeCategoryPath ( categoryPathMap . get ( defaultCategoryId ) ) ;
if ( ! StringUtil . hasText ( defaultCategoryPath ) ) {
defaultCategoryPath = DEFAULT_CATEGORY_NAME ;
}
Map < String , BigInteger > categoryPathIdMap = buildCategoryPathIdMap ( categoryPathMap ) ;
categoryPathIdMap . putIfAbsent ( defaultCategoryPath , defaultCategoryId ) ;
Map < String , FaqItem > existingFaqMap = buildExistingFaqMap ( collectionId , categoryPathMap , defaultCategoryPath ) ;
int nextOrderNo = nextOrderNo ( collectionId ) ;
TransactionTemplate rowTransactionTemplate = createImportRowTransactionTemplate ( ) ;
List < FaqImportRow > importRows = parseExcelRows ( file ) ;
FaqImportResultVo result = new FaqImportResultVo ( ) ;
result . setTotalCount ( importRows . size ( ) ) ;
for ( FaqImportRow row : importRows ) {
String question = trimToNull ( row . getQuestion ( ) ) ;
String answerHtml = trimToNull ( row . getAnswerHtml ( ) ) ;
if ( ! StringUtil . hasText ( question ) ) {
appendImportError ( result , row , " 问题不能为空 " ) ;
continue ;
}
if ( ! StringUtil . hasText ( answerHtml ) ) {
appendImportError ( result , row , " 答案HTML不能为空 " ) ;
continue ;
}
String inputPath = normalizeCategoryPath ( row . getCategoryPath ( ) ) ;
String resolvedCategoryPath = StringUtil . hasText ( inputPath ) ? inputPath : defaultCategoryPath ;
BigInteger resolvedCategoryId = categoryPathIdMap . get ( resolvedCategoryPath ) ;
if ( resolvedCategoryId = = null ) {
resolvedCategoryId = defaultCategoryId ;
resolvedCategoryPath = defaultCategoryPath ;
}
String upsertKey = buildImportUpsertKey ( resolvedCategoryPath , question ) ;
FaqItem matchedFaq = existingFaqMap . get ( upsertKey ) ;
Integer targetOrderNo = matchedFaq = = null ? nextOrderNo : matchedFaq . getOrderNo ( ) ;
if ( matchedFaq = = null ) {
nextOrderNo + + ;
}
try {
if ( matchedFaq = = null ) {
FaqItem saveEntity = new FaqItem ( ) ;
saveEntity . setCollectionId ( collectionId ) ;
saveEntity . setCategoryId ( resolvedCategoryId ) ;
saveEntity . setQuestion ( question ) ;
saveEntity . setAnswerHtml ( answerHtml ) ;
saveEntity . setOrderNo ( targetOrderNo ) ;
executeInRowTransaction ( rowTransactionTemplate , ( ) - > saveFaqItem ( saveEntity ) ) ;
existingFaqMap . put ( upsertKey , saveEntity ) ;
} else {
FaqItem updateEntity = new FaqItem ( ) ;
updateEntity . setId ( matchedFaq . getId ( ) ) ;
updateEntity . setCollectionId ( collectionId ) ;
updateEntity . setCategoryId ( resolvedCategoryId ) ;
updateEntity . setQuestion ( question ) ;
updateEntity . setAnswerHtml ( answerHtml ) ;
updateEntity . setOrderNo ( targetOrderNo ) ;
executeInRowTransaction ( rowTransactionTemplate , ( ) - > updateFaqItem ( updateEntity ) ) ;
matchedFaq . setCategoryId ( resolvedCategoryId ) ;
matchedFaq . setQuestion ( question ) ;
matchedFaq . setAnswerHtml ( answerHtml ) ;
matchedFaq . setOrderNo ( targetOrderNo ) ;
existingFaqMap . put ( upsertKey , matchedFaq ) ;
}
result . setSuccessCount ( result . getSuccessCount ( ) + 1 ) ;
} catch ( Exception e ) {
appendImportError ( result , row , extractImportErrorMessage ( e ) ) ;
}
}
result . setErrorCount ( result . getErrorRows ( ) . size ( ) ) ;
return result ;
}
@Override
public void exportToExcel ( BigInteger collectionId , OutputStream outputStream ) {
getFaqCollection ( collectionId ) ;
faqCategoryService . ensureDefaultCategory ( collectionId ) ;
Map < BigInteger , String > categoryPathMap = faqCategoryService . buildPathMap ( collectionId ) ;
List < FaqItem > faqItems = list ( QueryWrapper . create ( )
. eq ( FaqItem : : getCollectionId , collectionId )
. orderBy ( " order_no asc, id asc " ) ) ;
List < List < String > > rows = new ArrayList < > ( ) ;
for ( FaqItem faqItem : faqItems ) {
List < String > row = new ArrayList < > ( 3 ) ;
row . add ( defaultIfNull ( normalizeCategoryPath ( categoryPathMap . get ( faqItem . getCategoryId ( ) ) ) , DEFAULT_CATEGORY_NAME ) ) ;
row . add ( defaultIfNull ( faqItem . getQuestion ( ) , " " ) ) ;
row . add ( defaultIfNull ( faqItem . getAnswerHtml ( ) , " " ) ) ;
rows . add ( row ) ;
}
EasyExcel . write ( outputStream )
. head ( buildImportHeadList ( ) )
. registerWriteHandler ( new SimpleColumnWidthStyleStrategy ( TEMPLATE_COLUMN_WIDTH ) )
. sheet ( " FAQ " )
. doWrite ( rows ) ;
}
@Override
public void writeImportTemplate ( OutputStream outputStream ) {
ExcelWriter excelWriter = EasyExcel . write ( outputStream ) . build ( ) ;
try {
WriteSheet templateSheet = EasyExcel . writerSheet ( 0 , " 模板 " )
. head ( buildImportHeadList ( ) )
. registerWriteHandler ( new SimpleColumnWidthStyleStrategy ( TEMPLATE_COLUMN_WIDTH ) )
. build ( ) ;
excelWriter . write ( new ArrayList < > ( ) , templateSheet ) ;
WriteSheet instructionSheet = EasyExcel . writerSheet ( 1 , " 填写说明 " )
. head ( Collections . singletonList ( Collections . singletonList ( " 规则说明 " ) ) )
. registerWriteHandler ( new SimpleColumnWidthStyleStrategy ( TEMPLATE_NOTE_COLUMN_WIDTH ) )
. build ( ) ;
excelWriter . write ( buildImportInstructionRows ( ) , instructionSheet ) ;
} finally {
excelWriter . finish ( ) ;
}
}
private void checkAndNormalize ( FaqItem entity , boolean isSave , Map < String , Object > baseOptions ) {
if ( entity = = null ) {
throw new BusinessException ( " FAQ条目不能为空 " ) ;
@@ -177,6 +333,9 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
}
private DocumentCollection getFaqCollection ( BigInteger collectionId ) {
if ( collectionId = = null ) {
throw new BusinessException ( " 知识库ID不能为空 " ) ;
}
DocumentCollection collection = documentCollectionService . getById ( collectionId ) ;
if ( collection = = null ) {
throw new BusinessException ( " 知识库不存在 " ) ;
@@ -321,6 +480,159 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
}
}
private List < FaqImportRow > parseExcelRows ( MultipartFile file ) {
try ( InputStream inputStream = file . getInputStream ( ) ) {
FaqExcelReadListener listener = new FaqExcelReadListener ( ) ;
FastExcel . read ( inputStream , listener )
. sheet ( )
. doRead ( ) ;
return listener . getRows ( ) ;
} catch ( BusinessException e ) {
throw e ;
} catch ( Exception e ) {
throw new BusinessException ( " FAQ Excel解析失败 " ) ;
}
}
private void validateImportFile ( MultipartFile file ) {
if ( file = = null | | file . isEmpty ( ) ) {
throw new BusinessException ( " 导入文件不能为空 " ) ;
}
if ( file . getSize ( ) > MAX_IMPORT_FILE_SIZE_BYTES ) {
throw new BusinessException ( " 导入文件大小不能超过10MB " ) ;
}
String fileName = file . getOriginalFilename ( ) ;
String lowerName = fileName = = null ? " " : fileName . toLowerCase ( ) ;
if ( ! ( lowerName . endsWith ( " .xlsx " ) | | lowerName . endsWith ( " .xls " ) ) ) {
throw new BusinessException ( " 仅支持 xlsx/xls 文件 " ) ;
}
}
private List < List < String > > buildImportHeadList ( ) {
List < List < String > > headList = new ArrayList < > ( 3 ) ;
headList . add ( Collections . singletonList ( IMPORT_HEAD_CATEGORY_PATH ) ) ;
headList . add ( Collections . singletonList ( IMPORT_HEAD_QUESTION ) ) ;
headList . add ( Collections . singletonList ( IMPORT_HEAD_ANSWER_HTML ) ) ;
return headList ;
}
private List < List < String > > buildImportInstructionRows ( ) {
List < List < String > > rows = new ArrayList < > ( ) ;
rows . add ( Collections . singletonList ( " 1. 模板页为第一个Sheet( 名称: 模板) , 导入时将读取该Sheet。 " ) ) ;
rows . add ( Collections . singletonList ( " 2. 分类路径可为空;为空或不存在时会自动回落到“默认分类”。 " ) ) ;
rows . add ( Collections . singletonList ( " 3. 分类路径分隔符使用“/”,示例:产品/账号/登录。 " ) ) ;
rows . add ( Collections . singletonList ( " 4. 问题不能为空; 答案HTML不能为空。 " ) ) ;
rows . add ( Collections . singletonList ( " 5. 导入按“分类路径 + 问题”执行更新/新增( Upsert) 。 " ) ) ;
rows . add ( Collections . singletonList ( " 6. 同一个Excel里若键重复, 后出现的行会覆盖前面的行。 " ) ) ;
rows . add ( Collections . singletonList ( " 7. 单次最多导入5000条数据, 文件大小不超过10MB。 " ) ) ;
return rows ;
}
private void appendImportError ( FaqImportResultVo result , FaqImportRow row , String reason ) {
FaqImportErrorRowVo errorRow = new FaqImportErrorRowVo ( ) ;
errorRow . setRowNumber ( row . getRowNumber ( ) ) ;
errorRow . setCategoryPath ( row . getCategoryPath ( ) ) ;
errorRow . setQuestion ( row . getQuestion ( ) ) ;
errorRow . setReason ( reason ) ;
result . getErrorRows ( ) . add ( errorRow ) ;
}
private String extractImportErrorMessage ( Exception e ) {
if ( e = = null ) {
return " 导入失败 " ;
}
if ( e . getCause ( ) ! = null & & StringUtil . hasText ( e . getCause ( ) . getMessage ( ) ) ) {
return e . getCause ( ) . getMessage ( ) ;
}
if ( StringUtil . hasText ( e . getMessage ( ) ) ) {
return e . getMessage ( ) ;
}
return " 导入失败 " ;
}
private TransactionTemplate createImportRowTransactionTemplate ( ) {
TransactionTemplate template = new TransactionTemplate ( transactionManager ) ;
template . setPropagationBehavior ( TransactionDefinition . PROPAGATION_REQUIRES_NEW ) ;
return template ;
}
private boolean executeInRowTransaction ( TransactionTemplate template , Supplier < Boolean > action ) {
Boolean success = template . execute ( status - > {
boolean saved = action . get ( ) ;
if ( ! saved ) {
status . setRollbackOnly ( ) ;
return false ;
}
return true ;
} ) ;
if ( ! Boolean . TRUE . equals ( success ) ) {
throw new BusinessException ( " FAQ导入保存失败 " ) ;
}
return true ;
}
private Map < String , BigInteger > buildCategoryPathIdMap ( Map < BigInteger , String > categoryPathMap ) {
Map < String , BigInteger > pathIdMap = new HashMap < > ( ) ;
for ( Map . Entry < BigInteger , String > entry : categoryPathMap . entrySet ( ) ) {
if ( entry . getKey ( ) = = null ) {
continue ;
}
String normalizedPath = normalizeCategoryPath ( entry . getValue ( ) ) ;
if ( ! StringUtil . hasText ( normalizedPath ) ) {
continue ;
}
pathIdMap . put ( normalizedPath , entry . getKey ( ) ) ;
}
return pathIdMap ;
}
private Map < String , FaqItem > buildExistingFaqMap ( BigInteger collectionId ,
Map < BigInteger , String > categoryPathMap ,
String defaultCategoryPath ) {
Map < String , FaqItem > existingFaqMap = new HashMap < > ( ) ;
List < FaqItem > existingFaqItems = list ( QueryWrapper . create ( )
. eq ( FaqItem : : getCollectionId , collectionId )
. orderBy ( " order_no asc, id asc " ) ) ;
for ( FaqItem faqItem : existingFaqItems ) {
if ( faqItem . getId ( ) = = null | | StringUtil . noText ( faqItem . getQuestion ( ) ) ) {
continue ;
}
String categoryPath = normalizeCategoryPath ( categoryPathMap . get ( faqItem . getCategoryId ( ) ) ) ;
if ( ! StringUtil . hasText ( categoryPath ) ) {
categoryPath = defaultCategoryPath ;
}
String upsertKey = buildImportUpsertKey ( categoryPath , faqItem . getQuestion ( ) ) ;
existingFaqMap . putIfAbsent ( upsertKey , faqItem ) ;
}
return existingFaqMap ;
}
private String buildImportUpsertKey ( String categoryPath , String question ) {
return defaultIfNull ( categoryPath , DEFAULT_CATEGORY_NAME ) + IMPORT_UPSERT_KEY_SEPARATOR + question . trim ( ) ;
}
private String normalizeCategoryPath ( String categoryPath ) {
if ( ! StringUtil . hasText ( categoryPath ) ) {
return null ;
}
String [ ] nodes = categoryPath . trim ( ) . split ( " / " ) ;
List < String > normalized = new ArrayList < > ( ) ;
for ( String node : nodes ) {
String trimmed = trimToNull ( node ) ;
if ( trimmed ! = null ) {
normalized . add ( trimmed ) ;
}
}
if ( normalized . isEmpty ( ) ) {
return null ;
}
return String . join ( " / " , normalized ) ;
}
private String defaultIfNull ( String value , String fallback ) {
return value = = null ? fallback : value ;
}
private Integer nextOrderNo ( BigInteger collectionId ) {
java . util . List < FaqItem > list = list ( QueryWrapper . create ( )
. eq ( FaqItem : : getCollectionId , collectionId )
@@ -454,6 +766,106 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
return value . trim ( ) ;
}
private static class FaqImportRow {
private Integer rowNumber ;
private String categoryPath ;
private String question ;
private String answerHtml ;
public Integer getRowNumber ( ) {
return rowNumber ;
}
public void setRowNumber ( Integer rowNumber ) {
this . rowNumber = rowNumber ;
}
public String getCategoryPath ( ) {
return categoryPath ;
}
public void setCategoryPath ( String categoryPath ) {
this . categoryPath = categoryPath ;
}
public String getQuestion ( ) {
return question ;
}
public void setQuestion ( String question ) {
this . question = question ;
}
public String getAnswerHtml ( ) {
return answerHtml ;
}
public void setAnswerHtml ( String answerHtml ) {
this . answerHtml = answerHtml ;
}
}
private class FaqExcelReadListener implements ReadListener < LinkedHashMap < Integer , Object > > {
private final Map < String , Integer > headIndex = new HashMap < > ( ) ;
private final List < FaqImportRow > rows = new ArrayList < > ( ) ;
private int sheetRowNo ;
@Override
public void invoke ( LinkedHashMap < Integer , Object > data , AnalysisContext context ) {
sheetRowNo + + ;
String categoryPath = getCellValue ( data , IMPORT_HEAD_CATEGORY_PATH ) ;
String question = getCellValue ( data , IMPORT_HEAD_QUESTION ) ;
String answerHtml = getCellValue ( data , IMPORT_HEAD_ANSWER_HTML ) ;
if ( ! StringUtil . hasText ( categoryPath ) & & ! StringUtil . hasText ( question ) & & ! StringUtil . hasText ( answerHtml ) ) {
return ;
}
if ( rows . size ( ) > = MAX_IMPORT_ROWS ) {
throw new BusinessException ( " 单次最多导入5000条FAQ " ) ;
}
FaqImportRow row = new FaqImportRow ( ) ;
row . setRowNumber ( sheetRowNo + 1 ) ;
row . setCategoryPath ( categoryPath ) ;
row . setQuestion ( question ) ;
row . setAnswerHtml ( answerHtml ) ;
rows . add ( row ) ;
}
@Override
public void invokeHead ( Map < Integer , ReadCellData < ? > > headMap , AnalysisContext context ) {
for ( Map . Entry < Integer , ReadCellData < ? > > entry : headMap . entrySet ( ) ) {
String headValue = entry . getValue ( ) = = null ? null : entry . getValue ( ) . getStringValue ( ) ;
String header = trimToNull ( headValue ) ;
if ( header = = null ) {
continue ;
}
headIndex . put ( header , entry . getKey ( ) ) ;
}
if ( ! headIndex . containsKey ( IMPORT_HEAD_CATEGORY_PATH )
| | ! headIndex . containsKey ( IMPORT_HEAD_QUESTION )
| | ! headIndex . containsKey ( IMPORT_HEAD_ANSWER_HTML ) ) {
throw new BusinessException ( " 导入模板表头不正确, 必须包含: 分类路径、问题、答案HTML " ) ;
}
}
@Override
public void doAfterAllAnalysed ( AnalysisContext context ) {
// no-op
}
public List < FaqImportRow > getRows ( ) {
return rows ;
}
private String getCellValue ( Map < Integer , Object > row , String headName ) {
Integer index = headIndex . get ( headName ) ;
if ( index = = null ) {
return null ;
}
Object value = row . get ( index ) ;
return value = = null ? null : String . valueOf ( value ) . trim ( ) ;
}
}
private static class PreparedStore {
private final DocumentStore documentStore ;
private final StoreOptions storeOptions ;