feat: 完善知识库批量导入与公共 API

- 新增批量异步导入、状态查询、失败重试与中断恢复链路

- 拆分知识库读取、导入、维护权限并完善 Public API 契约

- 补充数据库迁移、管理端交互、接口说明与相关测试
This commit is contained in:
2026-08-03 11:13:48 +08:00
parent 6df3dd9981
commit 51dbfd41b6
99 changed files with 16274 additions and 480 deletions

View File

@@ -1,14 +1,33 @@
<script setup lang="ts">
import { ref } from 'vue';
import { computed, h, ref, watch } from 'vue';
import { useAppConfig } from '@easyflow/hooks';
import { $t } from '@easyflow/locales';
import { useAccessStore } from '@easyflow/stores';
import { ElButton, ElProgress, ElTable, ElTableColumn } from 'element-plus';
import { FolderOpened, UploadFilled } from '@element-plus/icons-vue';
import {
ElButton,
ElIcon,
ElMessage,
ElProgress,
ElTable,
ElTableColumn,
} from 'element-plus';
import {
ElTableV2,
ElAutoResizer as TableV2AutoResizer,
} from 'element-plus/es/components/table-v2/index.mjs';
import { formatFileSize } from '#/api/common/file';
import { api } from '#/api/request';
import DragFileUpload from '#/components/upload/DragFileUpload.vue';
interface FileInfo {
import { resolveDocumentUploadResponse } from './document-import-upload-response';
import 'element-plus/es/components/table-v2/style/css.mjs';
interface LegacyFileInfo {
uid: string;
fileName: string;
progressUpload: number;
@@ -16,99 +35,607 @@ interface FileInfo {
status: string;
filePath: string;
}
const fileData = ref<FileInfo[]>([]);
const filesPath = ref([]);
interface BatchFileInfo {
clientFileKey: string;
error?: string;
file: File;
fileName: string;
fileSize: number;
itemId?: string;
progressUpload: number;
relativePath: string;
status: 'error' | 'queued' | 'success' | 'uploading';
uid: string;
}
interface BatchCreateResponse {
batchId: string;
items: Array<{
clientFileKey: string;
itemId: string;
}>;
uploadConcurrency: number;
}
const props = defineProps({
batchMode: {
type: Boolean,
default: false,
},
knowledgeId: {
type: String,
default: '',
},
});
const emit = defineEmits<{
batchStateChange: [
state: {
ready: boolean;
uploading: boolean;
},
];
}>();
const MAX_FILE_COUNT = 2000;
const MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024;
const MAX_TOTAL_SIZE_BYTES = 1024 * 1024 * 1024;
const SUPPORTED_EXTENSIONS = new Set([
'docx',
'md',
'pdf',
'pptx',
'txt',
'xlsx',
]);
const fileData = ref<LegacyFileInfo[]>([]);
const filesPath = ref<any[]>([]);
const dragUploadRef = ref<InstanceType<typeof DragFileUpload>>();
const batchFiles = ref<BatchFileInfo[]>([]);
const batchId = ref('');
const folderInputRef = ref<HTMLInputElement>();
const fileInputRef = ref<HTMLInputElement>();
const activeRequests = new Set<XMLHttpRequest>();
const accessStore = useAccessStore();
const { apiURL } = useAppConfig(import.meta.env, import.meta.env.PROD);
let batchGeneration = 0;
const batchUploading = computed(() =>
batchFiles.value.some(
(item) => item.status === 'queued' || item.status === 'uploading',
),
);
const batchReady = computed(
() =>
Boolean(batchId.value) &&
batchFiles.value.length > 0 &&
batchFiles.value.every((item) => item.status === 'success'),
);
watch(
[batchReady, batchUploading],
([ready, uploading]) => {
emit('batchStateChange', { ready, uploading });
},
{ immediate: true },
);
function resolveUploadProgressStatus(
status: BatchFileInfo['status'],
): 'exception' | 'success' | undefined {
if (status === 'success') return 'success';
if (status === 'error') return 'exception';
return undefined;
}
const batchColumns = computed(() => [
{
cellRenderer: ({ rowData }: any) =>
h(
'span',
{
class: 'batch-file-name',
title: rowData.relativePath,
},
rowData.relativePath,
),
dataKey: 'relativePath',
flexGrow: 1,
key: 'relativePath',
title: $t('documentCollection.importDoc.fileName'),
width: 420,
},
{
cellRenderer: ({ rowData }: any) =>
h(ElProgress, {
percentage: Math.round(rowData.progressUpload || 0),
status: resolveUploadProgressStatus(rowData.status),
strokeWidth: 8,
}),
dataKey: 'progressUpload',
key: 'progressUpload',
title: $t('documentCollection.importDoc.progressUpload'),
width: 220,
},
{
cellRenderer: ({ rowData }: any) =>
h('span', formatFileSize(rowData.fileSize)),
dataKey: 'fileSize',
key: 'fileSize',
title: $t('documentCollection.importDoc.fileSize'),
width: 132,
},
{
cellRenderer: ({ rowData }: any) => {
if (rowData.status === 'error') {
return h(
ElButton,
{
link: true,
title:
rowData.error || $t('documentCollection.importDoc.uploadFailed'),
type: 'primary',
onClick: () => retryBatchUpload(rowData),
},
() => $t('documentCollection.importDoc.retryUpload'),
);
}
return h(
'span',
{
class: rowData.status === 'success' ? 'upload-state--success' : '',
title: rowData.error || '',
},
rowData.status === 'success'
? $t('documentCollection.importDoc.uploaded')
: $t('documentCollection.importDoc.uploading'),
);
},
dataKey: 'status',
key: 'status',
title: $t('common.handle'),
width: 112,
},
]);
function resetState() {
batchGeneration += 1;
for (const request of activeRequests) {
request.abort();
}
activeRequests.clear();
fileData.value = [];
filesPath.value = [];
batchFiles.value = [];
batchId.value = '';
dragUploadRef.value?.clearFiles?.();
if (folderInputRef.value) folderInputRef.value.value = '';
if (fileInputRef.value) fileInputRef.value.value = '';
}
async function cancelCurrentBatch() {
for (const request of activeRequests) {
request.abort();
}
activeRequests.clear();
const currentBatchId = batchId.value;
if (currentBatchId) {
const response = await api.post('/api/v1/document/import/batch/cancel', {
batchId: currentBatchId,
knowledgeId: props.knowledgeId,
});
if (response.errorCode !== 0) {
throw new Error(
response.message ||
$t('documentCollection.importDoc.cancelBatchFailed'),
);
}
}
resetState();
}
defineExpose({
cancelCurrentBatch,
getBatchId() {
return batchId.value;
},
getFilesData() {
return fileData.value.filter((item) => item.filePath);
},
reset() {
fileData.value = [];
filesPath.value = [];
dragUploadRef.value?.clearFiles?.();
isBatchReady() {
return batchReady.value;
},
isUploading() {
return batchUploading.value;
},
reset: resetState,
});
function handleSuccess(response: any) {
filesPath.value = response.data;
}
function handleChange(file: any) {
const existingFile = fileData.value.find((item) => item.uid === file.uid);
if (existingFile) {
fileData.value = fileData.value.map((item) => {
if (item.uid === file.uid) {
return {
...item,
fileSize: file.size,
progressUpload: file.percentage,
status: file.status,
filePath: file?.response?.data?.path,
};
}
return item;
});
} else {
fileData.value.push({
uid: file.uid,
fileName: file.name,
progressUpload: file.percentage,
fileSize: file.size,
status: file.status,
filePath: file?.response?.data?.path,
});
fileData.value = fileData.value.map((item) =>
item.uid === file.uid
? {
...item,
filePath: file?.response?.data?.path,
fileSize: file.size,
progressUpload: file.percentage,
status: file.status,
}
: item,
);
return;
}
fileData.value.push({
fileName: file.name,
filePath: file?.response?.data?.path,
fileSize: file.size,
progressUpload: file.percentage,
status: file.status,
uid: file.uid,
});
}
function handleRemove(row: LegacyFileInfo) {
fileData.value = fileData.value.filter((item) => item.uid !== row.uid);
}
function triggerFileSelect() {
if (!batchUploading.value) {
fileInputRef.value?.click();
}
}
function handleRemove(row: any) {
fileData.value = fileData.value.filter((item) => item.uid !== row.uid);
function triggerFolderSelect(event: Event) {
event.stopPropagation();
if (!batchUploading.value) {
folderInputRef.value?.click();
}
}
async function handleNativeSelection(event: Event) {
const input = event.target as HTMLInputElement;
const files = [...(input.files || [])];
input.value = '';
if (files.length > 0) {
await prepareBatch(files);
}
}
async function handleDrop(event: DragEvent) {
if (batchUploading.value) return;
const files = [...(event.dataTransfer?.files || [])];
if (files.length > 0) {
await prepareBatch(files);
}
}
async function prepareBatch(files: File[]) {
const generation = ++batchGeneration;
const accepted: File[] = [];
let totalBytes = 0;
let ignoredCount = 0;
for (const file of files) {
const relativePath = getRelativePath(file);
if (
relativePath.includes('/__MACOSX/') ||
relativePath.split('/').at(-1) === '.DS_Store'
) {
ignoredCount++;
continue;
}
const extension = file.name.split('.').pop()?.toLowerCase() || '';
if (!SUPPORTED_EXTENSIONS.has(extension)) {
ignoredCount++;
continue;
}
if (file.size > MAX_FILE_SIZE_BYTES) {
ElMessage.warning($t('documentCollection.importDoc.singleFileLimit'));
return;
}
totalBytes += file.size;
accepted.push(file);
}
if (accepted.length === 0) {
ElMessage.warning($t('documentCollection.importDoc.noSupportedFiles'));
return;
}
if (accepted.length > MAX_FILE_COUNT) {
ElMessage.warning($t('documentCollection.importDoc.fileCountLimit'));
return;
}
if (totalBytes > MAX_TOTAL_SIZE_BYTES) {
ElMessage.warning($t('documentCollection.importDoc.folderSizeLimit'));
return;
}
if (ignoredCount > 0) {
ElMessage.info($t('documentCollection.importDoc.unsupportedSkipped'));
}
batchFiles.value = await Promise.all(
accepted.map(async (file, index) => {
const relativePath = getRelativePath(file);
return {
clientFileKey: await createClientFileKey(relativePath),
file,
fileName: file.name,
fileSize: file.size,
progressUpload: 0,
relativePath,
status: 'queued',
uid: `${Date.now()}-${index}`,
} satisfies BatchFileInfo;
}),
);
batchId.value = '';
try {
const response = await api.post('/api/v1/document/import/batch/create', {
files: batchFiles.value.map((item) => ({
clientFileKey: item.clientFileKey,
fileName: item.fileName,
fileSize: item.fileSize,
relativePath: item.relativePath,
})),
knowledgeId: props.knowledgeId,
});
if (response.errorCode !== 0 || !response.data) {
throw new Error(
response.message ||
$t('documentCollection.importDoc.createBatchFailed'),
);
}
const data = response.data as BatchCreateResponse;
if (generation !== batchGeneration) {
await api.post('/api/v1/document/import/batch/cancel', {
batchId: data.batchId,
knowledgeId: props.knowledgeId,
});
return;
}
batchId.value = String(data.batchId);
const itemMap = new Map(
data.items.map((item) => [item.clientFileKey, String(item.itemId)]),
);
for (const item of batchFiles.value) {
item.itemId = itemMap.get(item.clientFileKey);
}
await uploadWithWorkers(Math.max(1, Number(data.uploadConcurrency || 3)));
} catch (error: any) {
for (const item of batchFiles.value) {
item.status = 'error';
item.error =
error?.message || $t('documentCollection.importDoc.createBatchFailed');
}
}
}
async function uploadWithWorkers(concurrency: number) {
let nextIndex = 0;
const worker = async () => {
while (nextIndex < batchFiles.value.length) {
const index = nextIndex++;
const item = batchFiles.value[index];
if (item) await uploadBatchItem(item);
}
};
await Promise.all(
Array.from({ length: Math.min(concurrency, batchFiles.value.length) }, () =>
worker(),
),
);
}
async function retryBatchUpload(item: BatchFileInfo) {
if (!item.itemId || !batchId.value) return;
await uploadBatchItem(item);
}
function uploadBatchItem(item: BatchFileInfo) {
return new Promise<void>((resolve) => {
if (!item.itemId || !batchId.value) {
item.status = 'error';
item.error = $t('documentCollection.importDoc.createBatchFailed');
resolve();
return;
}
const request = new XMLHttpRequest();
activeRequests.add(request);
item.status = 'uploading';
item.error = undefined;
request.open(
'POST',
`${apiURL}/api/v1/document/import/batch/${batchId.value}/item/${item.itemId}/upload?knowledgeId=${encodeURIComponent(props.knowledgeId)}`,
);
if (accessStore.accessToken) {
request.setRequestHeader('easyflow-token', accessStore.accessToken);
}
request.setRequestHeader('Accept', 'application/json');
request.upload.addEventListener('progress', (progressEvent) => {
if (progressEvent.lengthComputable) {
item.progressUpload = Math.round(
(progressEvent.loaded * 100) / progressEvent.total,
);
}
});
request.addEventListener('load', () => {
activeRequests.delete(request);
try {
const response = resolveDocumentUploadResponse({
fallbackMessage: $t('documentCollection.importDoc.uploadFailed'),
responseText: request.responseText || '',
status: request.status,
statusText: request.statusText,
});
if (!response.success) {
throw new Error(response.message);
}
item.progressUpload = 100;
item.status = 'success';
} catch (error: any) {
item.status = 'error';
item.error =
error?.message || $t('documentCollection.importDoc.uploadFailed');
}
resolve();
});
request.addEventListener('error', () => {
activeRequests.delete(request);
item.status = 'error';
item.error = $t('documentCollection.importDoc.uploadFailed');
resolve();
});
request.addEventListener('abort', () => {
activeRequests.delete(request);
resolve();
});
const formData = new FormData();
formData.append('file', item.file, item.fileName);
request.send(formData);
});
}
function getRelativePath(file: File) {
return (
(file as File & { webkitRelativePath?: string }).webkitRelativePath ||
file.name
).replaceAll('\\', '/');
}
async function createClientFileKey(relativePath: string) {
const source = relativePath;
if (globalThis.crypto?.subtle) {
const digest = await globalThis.crypto.subtle.digest(
'SHA-256',
new TextEncoder().encode(source),
);
return [...new Uint8Array(digest)]
.map((value) => value.toString(16).padStart(2, '0'))
.join('');
}
return [...source]
.reduce(
(hash, char) => (hash * 31 + (char.codePointAt(0) || 0)) >>> 0,
2_166_136_261,
)
.toString(16)
.padStart(64, '0');
}
</script>
<template>
<div class="import-file-container">
<DragFileUpload
ref="dragUploadRef"
@success="handleSuccess"
@on-change="handleChange"
/>
<div class="import-file-container__table">
<ElTable :data="fileData" style="width: 100%" size="large">
<ElTableColumn
prop="fileName"
:label="$t('documentCollection.importDoc.fileName')"
width="250"
/>
<ElTableColumn
prop="progressUpload"
:label="$t('documentCollection.importDoc.progressUpload')"
width="180"
<template v-if="batchMode">
<input
ref="fileInputRef"
class="native-file-input"
type="file"
multiple
accept=".txt,.pdf,.docx,.md,.pptx,.xlsx"
@change="handleNativeSelection"
/>
<input
ref="folderInputRef"
class="native-file-input"
type="file"
multiple
webkitdirectory
@change="handleNativeSelection"
/>
<div
class="batch-drop-zone"
:class="{ 'batch-drop-zone--disabled': batchUploading }"
role="button"
tabindex="0"
@click="triggerFileSelect"
@keydown.enter="triggerFileSelect"
@dragover.prevent
@drop.prevent="handleDrop"
>
<ElIcon class="batch-drop-zone__icon"><UploadFilled /></ElIcon>
<div class="batch-drop-zone__title">
{{ $t('documentCollection.importDoc.batchUploadTitle') }}
</div>
<div class="batch-drop-zone__description">
{{ $t('documentCollection.importDoc.batchUploadDescription') }}
</div>
<ElButton
:icon="FolderOpened"
:disabled="batchUploading"
@click="triggerFolderSelect"
>
<template #default="{ row }">
<ElProgress
:percentage="row.progressUpload"
v-if="row.status === 'success'"
status="success"
{{ $t('documentCollection.importDoc.selectFolder') }}
</ElButton>
</div>
<div class="batch-table">
<TableV2AutoResizer>
<template #default="{ height, width }">
<ElTableV2
:columns="batchColumns"
:data="batchFiles"
:height="height"
:width="width"
:row-height="52"
fixed
row-key="uid"
/>
<ElProgress v-else :percentage="row.progressUpload" />
</template>
</ElTableColumn>
<ElTableColumn
prop="fileSize"
:label="$t('documentCollection.importDoc.fileSize')"
>
<template #default="{ row }">
<span>{{ formatFileSize(row.fileSize) }}</span>
</template>
</ElTableColumn>
<ElTableColumn :label="$t('common.handle')">
<template #default="{ row }">
<ElButton type="danger" size="small" @click="handleRemove(row)">
{{ $t('button.delete') }}
</ElButton>
</template>
</ElTableColumn>
</ElTable>
</div>
</TableV2AutoResizer>
</div>
</template>
<template v-else>
<DragFileUpload
ref="dragUploadRef"
@success="handleSuccess"
@on-change="handleChange"
/>
<div class="import-file-container__table">
<ElTable :data="fileData" style="width: 100%" size="large">
<ElTableColumn
prop="fileName"
:label="$t('documentCollection.importDoc.fileName')"
width="250"
/>
<ElTableColumn
prop="progressUpload"
:label="$t('documentCollection.importDoc.progressUpload')"
width="180"
>
<template #default="{ row }">
<ElProgress
v-if="row.status === 'success'"
:percentage="row.progressUpload"
status="success"
/>
<ElProgress v-else :percentage="row.progressUpload" />
</template>
</ElTableColumn>
<ElTableColumn
prop="fileSize"
:label="$t('documentCollection.importDoc.fileSize')"
>
<template #default="{ row }">
<span>{{ formatFileSize(row.fileSize) }}</span>
</template>
</ElTableColumn>
<ElTableColumn :label="$t('common.handle')">
<template #default="{ row }">
<ElButton type="danger" size="small" @click="handleRemove(row)">
{{ $t('button.delete') }}
</ElButton>
</template>
</ElTableColumn>
</ElTable>
</div>
</template>
</div>
</template>
@@ -116,10 +643,80 @@ function handleRemove(row: any) {
.import-file-container {
display: flex;
flex-direction: column;
gap: 12px;
gap: 16px;
}
.import-file-container__table {
margin-top: 2px;
}
.native-file-input {
display: none;
}
.batch-drop-zone {
display: flex;
flex-direction: column;
gap: 8px;
align-items: center;
justify-content: center;
min-height: 208px;
padding: 24px;
cursor: pointer;
background: hsl(var(--surface-contrast-soft) / 54%);
border: 1px dashed var(--el-color-primary-light-5);
border-radius: 16px;
transition:
background-color 0.2s,
border-color 0.2s;
}
.batch-drop-zone:hover,
.batch-drop-zone:focus-visible {
background: var(--el-color-primary-light-9);
border-color: var(--el-color-primary);
outline: none;
}
.batch-drop-zone--disabled {
cursor: wait;
opacity: 0.72;
}
.batch-drop-zone__icon {
font-size: 48px;
color: var(--el-color-primary);
}
.batch-drop-zone__title {
font-size: 16px;
font-weight: 600;
color: var(--el-text-color-primary);
}
.batch-drop-zone__description {
margin-bottom: 4px;
font-size: 13px;
color: var(--el-text-color-secondary);
text-align: center;
}
.batch-table {
width: 100%;
height: 312px;
overflow: hidden;
border: 1px solid var(--el-border-color-lighter);
border-radius: 12px;
}
:deep(.batch-file-name) {
display: block;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
:deep(.upload-state--success) {
color: var(--el-color-success);
}
</style>