feat: 支持知识库 CSV 大文件导入

- 增加 CSV 流式解析、表格语义分块和分页预览

- 增加快照两阶段清理、失败重试和格式校验

- 补充批量入口、管理端交互和回归测试
This commit is contained in:
2026-08-07 13:11:59 +08:00
parent 13dec6c216
commit 0d14f1c165
32 changed files with 5672 additions and 156 deletions

View File

@@ -142,6 +142,7 @@ const isExcelChunk = (row: any) => {
return Boolean(
sourceFileExt === 'xlsx' ||
sourceFileExt === 'csv' ||
options?.sheetName ||
options?.rowStart ||
options?.rowEnd,

View File

@@ -81,6 +81,7 @@ const MAX_FILE_COUNT = 2000;
const MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024;
const MAX_TOTAL_SIZE_BYTES = 1024 * 1024 * 1024;
const SUPPORTED_EXTENSIONS = new Set([
'csv',
'docx',
'md',
'pdf',
@@ -537,7 +538,7 @@ async function createClientFileKey(relativePath: string) {
class="native-file-input"
type="file"
multiple
accept=".txt,.pdf,.docx,.md,.pptx,.xlsx"
accept=".txt,.pdf,.docx,.md,.pptx,.xlsx,.csv"
@change="handleNativeSelection"
/>
<input

View File

@@ -705,7 +705,7 @@ const endpointDocs = computed(() => {
'metadata 是 Content-Type 为 application/json 的 multipart Part不是普通表单字符串。',
'metadata.knowledgeId 填当前知识库 IDchunkStrategy 可省略duplicatePolicy 默认 SKIP。',
'metadata.files 中每项填写 clientFileKey、fileName 和可选 relativePath文件大小由服务端实际读取并统计。',
'支持 txt、md、pdf、docx、pptx、xlsx单文件不超过 100 MiB单批不超过 200 MiB最多 200 个文件。',
'支持 txt、md、pdf、docx、pptx、xlsx、csv;单文件不超过 100 MiB单批不超过 200 MiB最多 200 个文件。',
'metadata 最大 1 MiBmetadata.files 必须与 files Part 数量、顺序和文件名一致。',
'chunkStrategy 可不传,默认 AUTOduplicatePolicy 支持 SKIP、OVERWRITE、REIMPORT。',
'服务端会按调用者、元数据和文件内容生成请求指纹,任务完成后 10 分钟内的重复提交会返回原 taskId。',

View File

@@ -49,6 +49,8 @@ interface PreviewItem {
chunks?: ChunkItem[];
fileName: string;
normalizedContent?: string;
pageNo?: number;
pageSize?: number;
previewSessionId: string;
strategyCode?: string;
strategyLabel?: string;
@@ -97,6 +99,7 @@ const activeDocumentId = ref(props.documentId || '');
const previewError = ref('');
const previewLoading = ref(false);
const startLoading = ref(false);
const previewPageSize = 20;
let previewDebounceTimer: null | ReturnType<typeof setTimeout> = null;
let previewSequence = 0;
@@ -137,7 +140,9 @@ const fileExt = computed(
const isPptx = computed(() => fileExt.value === 'pptx');
const isXlsx = computed(() => fileExt.value === 'xlsx');
const showStrategySelector = computed(() => !isPptx.value && !isXlsx.value);
const isCsv = computed(() => fileExt.value === 'csv');
const isTabular = computed(() => isXlsx.value || isCsv.value);
const showStrategySelector = computed(() => !isPptx.value && !isTabular.value);
const mdLevels = [1, 2, 3, 4, 5, 6];
@@ -169,10 +174,10 @@ const buildStrategyConfig = () => {
strategyCode: 'OFFICE_PPTX_PAGE',
};
}
if (isXlsx.value) {
if (isTabular.value) {
return {
rowsPerChunk: formState.rowsPerChunk,
strategyCode: 'OFFICE_XLSX_ROW_WINDOW',
strategyCode: isCsv.value ? 'TABLE_ROW' : 'OFFICE_XLSX_ROW_WINDOW',
};
}
return {
@@ -233,6 +238,8 @@ const generatePreview = async () => {
},
],
knowledgeId: props.knowledgeId,
pageNo: 1,
pageSize: previewPageSize,
},
);
if (requestSequence !== previewSequence) {
@@ -261,6 +268,52 @@ const generatePreview = async () => {
}
};
const loadPreviewPage = async (pageNo: number) => {
if (
!activeDocumentId.value ||
!currentPreviewSessionId.value ||
previewLoading.value
) {
return;
}
const requestSequence = ++previewSequence;
previewLoading.value = true;
previewError.value = '';
try {
const res = await props.requestClient.post(
buildKnowledgePath(
props.endpointPrefix,
'/api/v1/document/import/task/preview',
),
{
documentId: activeDocumentId.value,
knowledgeId: props.knowledgeId,
pageNo,
pageSize: previewPageSize,
previewSessionId: currentPreviewSessionId.value,
},
);
if (requestSequence !== previewSequence) {
return;
}
previewItems.value = normalizePreviewItems(
(res.data?.items || []) as PreviewItem[],
);
} catch (error: any) {
if (requestSequence !== previewSequence) {
return;
}
const message =
error?.message || $t('documentCollection.importDoc.previewRequestFailed');
previewError.value = message;
ElMessage.error(message);
} finally {
if (requestSequence === previewSequence) {
previewLoading.value = false;
}
}
};
const schedulePreviewGeneration = () => {
if (!activeDocumentId.value) {
return;
@@ -321,8 +374,10 @@ watch(
if (isPptx.value) {
formState.strategyCode = 'OFFICE_PPTX_PAGE';
}
if (isXlsx.value) {
formState.strategyCode = 'OFFICE_XLSX_ROW_WINDOW';
if (isTabular.value) {
formState.strategyCode = isCsv.value
? 'TABLE_ROW'
: 'OFFICE_XLSX_ROW_WINDOW';
formState.rowsPerChunk = 10;
}
resetPreviewState();
@@ -365,7 +420,7 @@ watch(
</ElFormItem>
<ElFormItem
v-if="isXlsx"
v-if="isTabular"
label="每多少行分一块"
class="workbench__form-full"
>
@@ -450,6 +505,7 @@ watch(
<SplitterDocPreview
:loading="previewLoading"
:preview-items="previewItems"
@page-change="loadPreviewPage"
@preview-session-change="handlePreviewSessionChange"
/>
</section>

View File

@@ -4,7 +4,7 @@ import ElXMarkdown from 'vue-element-plus-x/es/XMarkdown/index.js';
import { $t } from '@easyflow/locales';
import { ElEmpty, ElSkeleton, ElTag } from 'element-plus';
import { ElEmpty, ElPagination, ElSkeleton, ElTag } from 'element-plus';
import {
markdownRenderProps,
@@ -42,6 +42,8 @@ interface PreviewItem {
chunks?: ChunkItem[];
fileName: string;
normalizedContent?: string;
pageNo?: number;
pageSize?: number;
previewSessionId: string;
strategyLabel?: string;
totalChunks?: number;
@@ -63,7 +65,7 @@ const props = withDefaults(
},
);
const emits = defineEmits(['previewSessionChange']);
const emits = defineEmits(['pageChange', 'previewSessionChange']);
const activeFile = ref('');
@@ -236,6 +238,23 @@ const isActiveChunk = (chunk: ChunkItem) =>
</div>
</div>
</div>
<div
v-if="
Number(currentPreview?.totalChunks || 0) >
Number(currentPreview?.pageSize || 20)
"
class="preview-shell__pagination"
>
<ElPagination
:current-page="currentPreview?.pageNo || 1"
:disabled="loading"
:page-size="currentPreview?.pageSize || 20"
:total="currentPreview?.totalChunks || 0"
background
layout="prev, pager, next"
@current-change="emits('pageChange', $event)"
/>
</div>
</div>
</div>
</template>
@@ -293,6 +312,13 @@ const isActiveChunk = (chunk: ChunkItem) =>
overflow: auto;
}
.preview-shell__pagination {
display: flex;
flex-shrink: 0;
justify-content: center;
padding-top: 16px;
}
.chunk-card {
padding: 15px 16px 14px;
cursor: pointer;