feat: 支持知识库 CSV 大文件导入
- 增加 CSV 流式解析、表格语义分块和分页预览 - 增加快照两阶段清理、失败重试和格式校验 - 补充批量入口、管理端交互和回归测试
This commit is contained in:
@@ -142,6 +142,7 @@ const isExcelChunk = (row: any) => {
|
||||
|
||||
return Boolean(
|
||||
sourceFileExt === 'xlsx' ||
|
||||
sourceFileExt === 'csv' ||
|
||||
options?.sheetName ||
|
||||
options?.rowStart ||
|
||||
options?.rowEnd,
|
||||
|
||||
@@ -81,6 +81,7 @@ const MAX_FILE_COUNT = 2000;
|
||||
const MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024;
|
||||
const MAX_TOTAL_SIZE_BYTES = 1024 * 1024 * 1024;
|
||||
const SUPPORTED_EXTENSIONS = new Set([
|
||||
'csv',
|
||||
'docx',
|
||||
'md',
|
||||
'pdf',
|
||||
@@ -537,7 +538,7 @@ async function createClientFileKey(relativePath: string) {
|
||||
class="native-file-input"
|
||||
type="file"
|
||||
multiple
|
||||
accept=".txt,.pdf,.docx,.md,.pptx,.xlsx"
|
||||
accept=".txt,.pdf,.docx,.md,.pptx,.xlsx,.csv"
|
||||
@change="handleNativeSelection"
|
||||
/>
|
||||
<input
|
||||
|
||||
@@ -705,7 +705,7 @@ const endpointDocs = computed(() => {
|
||||
'metadata 是 Content-Type 为 application/json 的 multipart Part,不是普通表单字符串。',
|
||||
'metadata.knowledgeId 填当前知识库 ID;chunkStrategy 可省略;duplicatePolicy 默认 SKIP。',
|
||||
'metadata.files 中每项填写 clientFileKey、fileName 和可选 relativePath;文件大小由服务端实际读取并统计。',
|
||||
'支持 txt、md、pdf、docx、pptx、xlsx;单文件不超过 100 MiB,单批不超过 200 MiB,最多 200 个文件。',
|
||||
'支持 txt、md、pdf、docx、pptx、xlsx、csv;单文件不超过 100 MiB,单批不超过 200 MiB,最多 200 个文件。',
|
||||
'metadata 最大 1 MiB;metadata.files 必须与 files Part 数量、顺序和文件名一致。',
|
||||
'chunkStrategy 可不传,默认 AUTO;duplicatePolicy 支持 SKIP、OVERWRITE、REIMPORT。',
|
||||
'服务端会按调用者、元数据和文件内容生成请求指纹,任务完成后 10 分钟内的重复提交会返回原 taskId。',
|
||||
|
||||
@@ -49,6 +49,8 @@ interface PreviewItem {
|
||||
chunks?: ChunkItem[];
|
||||
fileName: string;
|
||||
normalizedContent?: string;
|
||||
pageNo?: number;
|
||||
pageSize?: number;
|
||||
previewSessionId: string;
|
||||
strategyCode?: string;
|
||||
strategyLabel?: string;
|
||||
@@ -97,6 +99,7 @@ const activeDocumentId = ref(props.documentId || '');
|
||||
const previewError = ref('');
|
||||
const previewLoading = ref(false);
|
||||
const startLoading = ref(false);
|
||||
const previewPageSize = 20;
|
||||
let previewDebounceTimer: null | ReturnType<typeof setTimeout> = null;
|
||||
let previewSequence = 0;
|
||||
|
||||
@@ -137,7 +140,9 @@ const fileExt = computed(
|
||||
|
||||
const isPptx = computed(() => fileExt.value === 'pptx');
|
||||
const isXlsx = computed(() => fileExt.value === 'xlsx');
|
||||
const showStrategySelector = computed(() => !isPptx.value && !isXlsx.value);
|
||||
const isCsv = computed(() => fileExt.value === 'csv');
|
||||
const isTabular = computed(() => isXlsx.value || isCsv.value);
|
||||
const showStrategySelector = computed(() => !isPptx.value && !isTabular.value);
|
||||
|
||||
const mdLevels = [1, 2, 3, 4, 5, 6];
|
||||
|
||||
@@ -169,10 +174,10 @@ const buildStrategyConfig = () => {
|
||||
strategyCode: 'OFFICE_PPTX_PAGE',
|
||||
};
|
||||
}
|
||||
if (isXlsx.value) {
|
||||
if (isTabular.value) {
|
||||
return {
|
||||
rowsPerChunk: formState.rowsPerChunk,
|
||||
strategyCode: 'OFFICE_XLSX_ROW_WINDOW',
|
||||
strategyCode: isCsv.value ? 'TABLE_ROW' : 'OFFICE_XLSX_ROW_WINDOW',
|
||||
};
|
||||
}
|
||||
return {
|
||||
@@ -233,6 +238,8 @@ const generatePreview = async () => {
|
||||
},
|
||||
],
|
||||
knowledgeId: props.knowledgeId,
|
||||
pageNo: 1,
|
||||
pageSize: previewPageSize,
|
||||
},
|
||||
);
|
||||
if (requestSequence !== previewSequence) {
|
||||
@@ -261,6 +268,52 @@ const generatePreview = async () => {
|
||||
}
|
||||
};
|
||||
|
||||
const loadPreviewPage = async (pageNo: number) => {
|
||||
if (
|
||||
!activeDocumentId.value ||
|
||||
!currentPreviewSessionId.value ||
|
||||
previewLoading.value
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const requestSequence = ++previewSequence;
|
||||
previewLoading.value = true;
|
||||
previewError.value = '';
|
||||
try {
|
||||
const res = await props.requestClient.post(
|
||||
buildKnowledgePath(
|
||||
props.endpointPrefix,
|
||||
'/api/v1/document/import/task/preview',
|
||||
),
|
||||
{
|
||||
documentId: activeDocumentId.value,
|
||||
knowledgeId: props.knowledgeId,
|
||||
pageNo,
|
||||
pageSize: previewPageSize,
|
||||
previewSessionId: currentPreviewSessionId.value,
|
||||
},
|
||||
);
|
||||
if (requestSequence !== previewSequence) {
|
||||
return;
|
||||
}
|
||||
previewItems.value = normalizePreviewItems(
|
||||
(res.data?.items || []) as PreviewItem[],
|
||||
);
|
||||
} catch (error: any) {
|
||||
if (requestSequence !== previewSequence) {
|
||||
return;
|
||||
}
|
||||
const message =
|
||||
error?.message || $t('documentCollection.importDoc.previewRequestFailed');
|
||||
previewError.value = message;
|
||||
ElMessage.error(message);
|
||||
} finally {
|
||||
if (requestSequence === previewSequence) {
|
||||
previewLoading.value = false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const schedulePreviewGeneration = () => {
|
||||
if (!activeDocumentId.value) {
|
||||
return;
|
||||
@@ -321,8 +374,10 @@ watch(
|
||||
if (isPptx.value) {
|
||||
formState.strategyCode = 'OFFICE_PPTX_PAGE';
|
||||
}
|
||||
if (isXlsx.value) {
|
||||
formState.strategyCode = 'OFFICE_XLSX_ROW_WINDOW';
|
||||
if (isTabular.value) {
|
||||
formState.strategyCode = isCsv.value
|
||||
? 'TABLE_ROW'
|
||||
: 'OFFICE_XLSX_ROW_WINDOW';
|
||||
formState.rowsPerChunk = 10;
|
||||
}
|
||||
resetPreviewState();
|
||||
@@ -365,7 +420,7 @@ watch(
|
||||
</ElFormItem>
|
||||
|
||||
<ElFormItem
|
||||
v-if="isXlsx"
|
||||
v-if="isTabular"
|
||||
label="每多少行分一块"
|
||||
class="workbench__form-full"
|
||||
>
|
||||
@@ -450,6 +505,7 @@ watch(
|
||||
<SplitterDocPreview
|
||||
:loading="previewLoading"
|
||||
:preview-items="previewItems"
|
||||
@page-change="loadPreviewPage"
|
||||
@preview-session-change="handlePreviewSessionChange"
|
||||
/>
|
||||
</section>
|
||||
|
||||
@@ -4,7 +4,7 @@ import ElXMarkdown from 'vue-element-plus-x/es/XMarkdown/index.js';
|
||||
|
||||
import { $t } from '@easyflow/locales';
|
||||
|
||||
import { ElEmpty, ElSkeleton, ElTag } from 'element-plus';
|
||||
import { ElEmpty, ElPagination, ElSkeleton, ElTag } from 'element-plus';
|
||||
|
||||
import {
|
||||
markdownRenderProps,
|
||||
@@ -42,6 +42,8 @@ interface PreviewItem {
|
||||
chunks?: ChunkItem[];
|
||||
fileName: string;
|
||||
normalizedContent?: string;
|
||||
pageNo?: number;
|
||||
pageSize?: number;
|
||||
previewSessionId: string;
|
||||
strategyLabel?: string;
|
||||
totalChunks?: number;
|
||||
@@ -63,7 +65,7 @@ const props = withDefaults(
|
||||
},
|
||||
);
|
||||
|
||||
const emits = defineEmits(['previewSessionChange']);
|
||||
const emits = defineEmits(['pageChange', 'previewSessionChange']);
|
||||
|
||||
const activeFile = ref('');
|
||||
|
||||
@@ -236,6 +238,23 @@ const isActiveChunk = (chunk: ChunkItem) =>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
v-if="
|
||||
Number(currentPreview?.totalChunks || 0) >
|
||||
Number(currentPreview?.pageSize || 20)
|
||||
"
|
||||
class="preview-shell__pagination"
|
||||
>
|
||||
<ElPagination
|
||||
:current-page="currentPreview?.pageNo || 1"
|
||||
:disabled="loading"
|
||||
:page-size="currentPreview?.pageSize || 20"
|
||||
:total="currentPreview?.totalChunks || 0"
|
||||
background
|
||||
layout="prev, pager, next"
|
||||
@current-change="emits('pageChange', $event)"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -293,6 +312,13 @@ const isActiveChunk = (chunk: ChunkItem) =>
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.preview-shell__pagination {
|
||||
display: flex;
|
||||
flex-shrink: 0;
|
||||
justify-content: center;
|
||||
padding-top: 16px;
|
||||
}
|
||||
|
||||
.chunk-card {
|
||||
padding: 15px 16px 14px;
|
||||
cursor: pointer;
|
||||
|
||||
Reference in New Issue
Block a user