EasyFlow-Tech-Dev/EasyFlow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(faq): support rich-text image upload and RAG output

Browse Source 
- add FAQ image upload API with MinIO path and mime/size validation

- enable FAQ editor image upload and text-or-image validation

- include FAQ image metadata in vector content and retrieval output

- enforce markdown image preservation via bot system prompt rule
This commit is contained in:
陈子默
2026-02-25 19:53:39 +08:00
parent fcf1100b56
commit 01f354ede5
8 changed files with 396 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagents/tool/DocumentCollectionTool.java
View File

@@ -55,7 +55,14 @@ public class DocumentCollectionTool extends BaseTool {
StringBuilder sb = new StringBuilder();
if (documents != null) {
for (Document document : documents) {
for (int i = 0; i < documents.size(); i++) {
Document document = documents.get(i);
if (document == null || document.getContent() == null) {
continue;
}
if (sb.length() > 0) {
sb.append("\n\n---\n\n");
}
sb.append(document.getContent());
}
}

21
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/BotServiceImpl.java
View File

@@ -61,6 +61,7 @@ import static tech.easyflow.ai.entity.table.PluginItemTableDef.PLUGIN_ITEM;
public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotService {
private static final Logger log = LoggerFactory.getLogger(BotServiceImpl.class);
private static final String FAQ_IMAGE_SYSTEM_RULE = "当知识工具返回 Markdown 图片（格式：![描述](URL)）时，你必须在最终回答中保留并输出对应的图片 Markdown，禁止改写、替换或省略图片 URL。";
public static class ChatCheckResult {
private Bot aiBot;
@@ -173,7 +174,9 @@ public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotSe
Map<String, Object> modelOptions = chatCheckResult.getModelOptions();
ChatModel chatModel = chatCheckResult.getChatModel();
final MemoryPrompt memoryPrompt = new MemoryPrompt();
String systemPrompt = MapUtil.getString(modelOptions, Bot.KEY_SYSTEM_PROMPT);
String systemPrompt = buildSystemPromptWithFaqImageRule(
MapUtil.getString(modelOptions, Bot.KEY_SYSTEM_PROMPT)
);
Integer maxMessageCount = MapUtil.getInteger(modelOptions, Bot.KEY_MAX_MESSAGE_COUNT);
if (maxMessageCount != null) {
memoryPrompt.setMaxAttachedMessageCount(maxMessageCount);
@@ -221,6 +224,9 @@ public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotSe
Map<String, Object> modelOptions = chatCheckResult.getModelOptions();
ChatOptions chatOptions = getChatOptions(modelOptions);
ChatModel chatModel = chatCheckResult.getChatModel();
String systemPrompt = buildSystemPromptWithFaqImageRule(
MapUtil.getString(modelOptions, Bot.KEY_SYSTEM_PROMPT)
);
UserMessage userMessage = new UserMessage(prompt);
userMessage.addTools(buildFunctionList(Maps.of("botId", botId)
.set("needEnglishName", false)
@@ -231,6 +237,9 @@ public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotSe
ChatMemory defaultChatMemory = new PublicBotMessageMemory(chatSseEmitter, messages);
final MemoryPrompt memoryPrompt = new MemoryPrompt();
memoryPrompt.setMemory(defaultChatMemory);
if (StringUtils.hasLength(systemPrompt)) {
memoryPrompt.setSystemMessage(SystemMessage.of(systemPrompt));
}
memoryPrompt.addMessage(userMessage);
RequestAttributes requestAttributes = RequestContextHolder.getRequestAttributes();
ServletRequestAttributes sra = (ServletRequestAttributes) requestAttributes;
@@ -397,4 +406,14 @@ public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotSe
return messageBuilder.toString();
}
private String buildSystemPromptWithFaqImageRule(String systemPrompt) {
if (!StringUtils.hasLength(systemPrompt)) {
return FAQ_IMAGE_SYSTEM_RULE;
}
if (systemPrompt.contains(FAQ_IMAGE_SYSTEM_RULE)) {
return systemPrompt;
}
return systemPrompt + "\n\n" + FAQ_IMAGE_SYSTEM_RULE;
}
}

85
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java
View File

@@ -52,6 +52,7 @@ import static tech.easyflow.ai.entity.DocumentCollection.*;
public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectionMapper, DocumentCollection> implements DocumentCollectionService {
private static final Logger LOG = LoggerFactory.getLogger(DocumentCollectionServiceImpl.class);
private static final int MAX_FAQ_IMAGES_IN_PROMPT = 3;
@Resource
private ModelService llmService;
@@ -256,7 +257,18 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
searchDocuments.forEach(item -> {
FaqItem faqItem = faqItemMap.get(String.valueOf(item.getId()));
if (faqItem != null) {
item.setContent("问题：" + faqItem.getQuestion() + "\n答案：" + faqItem.getAnswerText());
List<Map<String, String>> faqImages = readFaqImages(faqItem);
item.setContent(buildFaqPromptContent(faqItem, faqImages));
Map<String, Object> metadataMap = item.getMetadataMap() == null
? new HashMap<>()
: new HashMap<>(item.getMetadataMap());
List<String> imageUrls = faqImages.stream()
.map(image -> image.get("url"))
.filter(Objects::nonNull)
.collect(Collectors.toList());
metadataMap.put("imageUrls", imageUrls);
item.setMetadataMap(metadataMap);
}
});
return;
@@ -271,4 +283,75 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
}
});
}
private String buildFaqPromptContent(FaqItem faqItem, List<Map<String, String>> images) {
StringBuilder contentBuilder = new StringBuilder();
contentBuilder.append("问题：").append(faqItem.getQuestion());
if (StringUtil.hasText(faqItem.getAnswerText())) {
contentBuilder.append("\n答案：").append(faqItem.getAnswerText());
}
if (images == null || images.isEmpty()) {
return contentBuilder.toString();
}
contentBuilder.append("\n相关图片：");
int limit = Math.min(images.size(), MAX_FAQ_IMAGES_IN_PROMPT);
for (int i = 0; i < limit; i++) {
Map<String, String> image = images.get(i);
String url = image.get("url");
if (!StringUtil.hasText(url)) {
continue;
}
String alt = image.get("alt");
if (!StringUtil.hasText(alt)) {
alt = image.get("title");
}
if (!StringUtil.hasText(alt)) {
alt = "faq-image-" + (i + 1);
}
contentBuilder.append("\n").append("![").append(alt).append("](").append(url).append(")");
}
return contentBuilder.toString();
}
@SuppressWarnings("unchecked")
private List<Map<String, String>> readFaqImages(FaqItem faqItem) {
if (faqItem == null || faqItem.getOptions() == null) {
return Collections.emptyList();
}
Object imagesValue = faqItem.getOptions().get("images");
if (!(imagesValue instanceof List<?>)) {
return Collections.emptyList();
}
List<Map<String, String>> images = new ArrayList<>();
for (Object imageObj : (List<Object>) imagesValue) {
if (!(imageObj instanceof Map<?, ?>)) {
continue;
}
Map<?, ?> imageMap = (Map<?, ?>) imageObj;
String url = trimToNull(imageMap.get("url"));
if (url == null) {
continue;
}
Map<String, String> normalized = new HashMap<>();
normalized.put("url", url);
String alt = trimToNull(imageMap.get("alt"));
if (alt != null) {
normalized.put("alt", alt);
}
String title = trimToNull(imageMap.get("title"));
if (title != null) {
normalized.put("title", title);
}
images.add(normalized);
}
return images;
}
private String trimToNull(Object value) {
if (value == null) {
return null;
}
String text = String.valueOf(value).trim();
return text.isEmpty() ? null : text;
}
}

159
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/FaqItemServiceImpl.java
View File

@@ -10,6 +10,8 @@ import com.easyagents.search.engine.service.DocumentSearcher;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.jsoup.safety.Safelist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +42,13 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
private static final Logger LOG = LoggerFactory.getLogger(FaqItemServiceImpl.class);
private static final Safelist ANSWER_SAFE_LIST = Safelist.basic()
.addTags("h1", "h2", "h3", "h4", "h5", "h6");
.addTags("h1", "h2", "h3", "h4", "h5", "h6", "img")
.addAttributes("img", "src", "alt", "title", "width", "height")
.addProtocols("img", "src", "http", "https");
private static final String OPTION_KEY_IMAGES = "images";
private static final String IMAGE_FIELD_URL = "url";
private static final String IMAGE_FIELD_ALT = "alt";
private static final String IMAGE_FIELD_TITLE = "title";
@Resource
private DocumentCollectionService documentCollectionService;
@@ -57,7 +65,7 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
@Override
@Transactional
public boolean saveFaqItem(FaqItem entity) {
checkAndNormalize(entity, true);
checkAndNormalize(entity, true, null);
DocumentCollection collection = getFaqCollection(entity.getCollectionId());
entity.setCategoryId(faqCategoryService.ensureFaqItemCategory(entity.getCollectionId(), entity.getCategoryId()));
@@ -93,7 +101,7 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
if (entity.getCollectionId() == null) {
entity.setCollectionId(old.getCollectionId());
}
checkAndNormalize(entity, false);
checkAndNormalize(entity, false, old.getOptions());
DocumentCollection collection = getFaqCollection(old.getCollectionId());
old.setCategoryId(faqCategoryService.ensureFaqItemCategory(old.getCollectionId(), entity.getCategoryId()));
@@ -128,7 +136,7 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
return removeById(id);
}
private void checkAndNormalize(FaqItem entity, boolean isSave) {
private void checkAndNormalize(FaqItem entity, boolean isSave, Map<String, Object> baseOptions) {
if (entity == null) {
throw new BusinessException("FAQ条目不能为空");
}
@@ -148,11 +156,24 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
entity.setQuestion(entity.getQuestion().trim());
String cleanHtml = Jsoup.clean(entity.getAnswerHtml(), ANSWER_SAFE_LIST);
String answerText = Jsoup.parse(cleanHtml).text();
if (StringUtil.noText(answerText)) {
List<Map<String, String>> images = extractImages(cleanHtml);
if (StringUtil.noText(answerText) && images.isEmpty()) {
throw new BusinessException("答案不能为空");
}
Map<String, Object> normalizedOptions = baseOptions == null
? new HashMap<>()
: new HashMap<>(baseOptions);
if (entity.getOptions() != null) {
normalizedOptions.putAll(entity.getOptions());
}
if (images.isEmpty()) {
normalizedOptions.remove(OPTION_KEY_IMAGES);
} else {
normalizedOptions.put(OPTION_KEY_IMAGES, images);
}
entity.setAnswerHtml(cleanHtml);
entity.setAnswerText(answerText);
entity.setOptions(normalizedOptions);
}
private DocumentCollection getFaqCollection(BigInteger collectionId) {
@@ -262,12 +283,22 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
metadata.put("question", entity.getQuestion());
metadata.put("answerText", entity.getAnswerText());
metadata.put("categoryId", entity.getCategoryId());
metadata.put("imageUrls", readImageUrls(entity.getOptions()));
doc.setMetadataMap(metadata);
return doc;
}
private String buildSearchContent(FaqItem entity) {
return "问题：" + entity.getQuestion() + "\n答案：" + entity.getAnswerText();
StringBuilder contentBuilder = new StringBuilder();
contentBuilder.append("问题：").append(entity.getQuestion());
if (StringUtil.hasText(entity.getAnswerText())) {
contentBuilder.append("\n答案：").append(entity.getAnswerText());
}
String imageDescription = buildImageDescriptionText(entity.getOptions());
if (StringUtil.hasText(imageDescription)) {
contentBuilder.append("\n图片说明：").append(imageDescription);
}
return contentBuilder.toString();
}
private void markCollectionEmbedded(DocumentCollection collection, EmbeddingModel embeddingModel) {
@@ -307,6 +338,122 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
return BigInteger.valueOf(StpUtil.getLoginIdAsLong());
}
private List<Map<String, String>> extractImages(String answerHtml) {
if (StringUtil.noText(answerHtml)) {
return Collections.emptyList();
}
Elements imageElements = Jsoup.parseBodyFragment(answerHtml).select("img[src]");
if (imageElements.isEmpty()) {
return Collections.emptyList();
}
List<Map<String, String>> images = new ArrayList<>();
Set<String> visited = new HashSet<>();
for (Element imageElement : imageElements) {
String url = trimToNull(imageElement.attr("src"));
if (url == null || !url.startsWith("http")) {
continue;
}
if (!visited.add(url)) {
continue;
}
Map<String, String> image = new LinkedHashMap<>();
image.put(IMAGE_FIELD_URL, url);
String alt = trimToNull(imageElement.attr("alt"));
if (alt != null) {
image.put(IMAGE_FIELD_ALT, alt);
}
String title = trimToNull(imageElement.attr("title"));
if (title != null) {
image.put(IMAGE_FIELD_TITLE, title);
}
images.add(image);
}
return images;
}
private String buildImageDescriptionText(Map<String, Object> options) {
List<Map<String, String>> images = normalizeImageField(options);
if (images.isEmpty()) {
return null;
}
List<String> descriptions = new ArrayList<>();
for (Map<String, String> image : images) {
String alt = trimToNull(image.get(IMAGE_FIELD_ALT));
if (alt != null) {
descriptions.add(alt);
continue;
}
String title = trimToNull(image.get(IMAGE_FIELD_TITLE));
if (title != null) {
descriptions.add(title);
}
}
if (descriptions.isEmpty()) {
return null;
}
return String.join("；", descriptions);
}
private List<String> readImageUrls(Map<String, Object> options) {
List<Map<String, String>> images = normalizeImageField(options);
if (images.isEmpty()) {
return Collections.emptyList();
}
List<String> imageUrls = new ArrayList<>();
for (Map<String, String> image : images) {
String url = trimToNull(image.get(IMAGE_FIELD_URL));
if (url != null) {
imageUrls.add(url);
}
}
return imageUrls;
}
@SuppressWarnings("unchecked")
private List<Map<String, String>> normalizeImageField(Map<String, Object> options) {
if (options == null) {
return Collections.emptyList();
}
Object imagesValue = options.get(OPTION_KEY_IMAGES);
if (!(imagesValue instanceof List<?>)) {
return Collections.emptyList();
}
List<Map<String, String>> normalized = new ArrayList<>();
for (Object imageObj : (List<Object>) imagesValue) {
if (!(imageObj instanceof Map<?, ?>)) {
continue;
}
Map<?, ?> imageMap = (Map<?, ?>) imageObj;
String url = trimToNull(toStringValue(imageMap.get(IMAGE_FIELD_URL)));
if (url == null) {
continue;
}
Map<String, String> image = new LinkedHashMap<>();
image.put(IMAGE_FIELD_URL, url);
String alt = trimToNull(toStringValue(imageMap.get(IMAGE_FIELD_ALT)));
if (alt != null) {
image.put(IMAGE_FIELD_ALT, alt);
}
String title = trimToNull(toStringValue(imageMap.get(IMAGE_FIELD_TITLE)));
if (title != null) {
image.put(IMAGE_FIELD_TITLE, title);
}
normalized.add(image);
}
return normalized;
}
private String toStringValue(Object value) {
return value == null ? null : String.valueOf(value);
}
private String trimToNull(String value) {
if (!StringUtil.hasText(value)) {
return null;
}
return value.trim();
}
private static class PreparedStore {
private final DocumentStore documentStore;
private final StoreOptions storeOptions;