diff --git a/backend/open_webui/routers/files.py b/backend/open_webui/routers/files.py index 8a09601b18..31157dd803 100644 --- a/backend/open_webui/routers/files.py +++ b/backend/open_webui/routers/files.py @@ -185,6 +185,7 @@ async def process_uploaded_file( request, ProcessFileForm(file_id=file_item.id, collection_name=knowledge_id), user=user, + db=db_session, ) log.info(f'Linked file {file_item.id} to knowledge {knowledge_id}') except Exception as e: diff --git a/src/lib/apis/files/index.ts b/src/lib/apis/files/index.ts index 37a49f7c7b..6c041452b7 100644 --- a/src/lib/apis/files/index.ts +++ b/src/lib/apis/files/index.ts @@ -5,7 +5,8 @@ export const uploadFile = async ( token: string, file: File, metadata?: object | null, - process?: boolean | null + process?: boolean | null, + stream: boolean = true ) => { const data = new FormData(); data.append('file', file); @@ -42,7 +43,7 @@ export const uploadFile = async ( throw error; } - if (res) { + if (res && stream) { const status = await getFileProcessStatus(token, res.id); if (status && status.ok) { diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index e92cb8fa6d..8042dbfe1b 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -78,7 +78,8 @@ let showNewDirectoryModal = false; let showSyncConfirmModal = false; - let pendingSyncFiles: Array<{path: string, filename: string, file: File}> | null = null; + let pendingSyncFiles: Array<{ path: string; filename: string; file: File }> | null = null; + let syncing: string | null = null; let showAccessControlModal = false; let minSize = 0; @@ -542,13 +543,17 @@ }; // Collect files from a directory without uploading — returns {path, filename, file}[] - const collectDirectoryFiles = async (): Promise | null> => { + const collectDirectoryFiles = async (): Promise | null> => { const isFileSystemAccessSupported = 'showDirectoryPicker' in window; try { if (isFileSystemAccessSupported) { const dirHandle = await window.showDirectoryPicker(); - const collected: Array<{path: string, filename: string, file: File}> = []; + const collected: Array<{ path: string; filename: string; file: File }> = []; async function traverse(handle: FileSystemDirectoryHandle, dirPath = '') { for await (const entry of handle.values()) { @@ -580,8 +585,9 @@ input.onchange = () => { try { - const files = Array.from(input.files || []) - .filter((file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.')); + const files = Array.from(input.files || []).filter( + (file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.') + ); const collected = files.map((file) => { const parts = file.webkitRelativePath.split('/'); @@ -618,92 +624,112 @@ const syncDirectoryHandler = async () => { if (!pendingSyncFiles?.length) return; - // ── 2. Compute checksums ── - toast.info($i18n.t('Computing checksums...')); - const manifest = await Promise.all( - pendingSyncFiles.map(async (entry) => ({ - ...entry, - checksum: await computeFileHash(entry.file), - size: entry.file.size - })) - ); - pendingSyncFiles = null; - - // ── 3. Diff against knowledge base ── - toast.info($i18n.t('Comparing with knowledge base...')); - const diff = await syncKnowledgeDiff( - localStorage.token, - id, - manifest.map(({ filename, path, checksum, size }) => ({ filename, path, checksum, size })) - ); - - if (!diff) { - toast.error($i18n.t('Failed to compare files.')); - return; - } - - // ── 4. mkdir — create missing directories (parents first) ── - const createdDirectoryIds: Record = {}; - for (const dirPath of diff.mkdir) { - const segments = dirPath.split('/'); - const name = segments.at(-1)!; - const parentPath = segments.slice(0, -1).join('/'); - const parentId = parentPath ? createdDirectoryIds[parentPath] : null; - - const directory = await createKnowledgeDirectory( - localStorage.token, knowledge.id, name, parentId - ); - if (directory) { - createdDirectoryIds[dirPath] = directory.id; - } - } - - // ── 5. Upload added + modified files ── - const filesToUpload = manifest.filter((entry) => - diff.added.some((a: any) => a.filename === entry.filename && a.path === entry.path) || - diff.modified.some((m: any) => m.filename === entry.filename && m.path === entry.path) - ); - - let uploadedCount = 0; - for (const entry of filesToUpload) { - const fileObject = new File([entry.file], entry.filename, { type: entry.file.type }); - await uploadFile(localStorage.token, fileObject, { - knowledge_id: knowledge.id, - file_hash: entry.checksum, - directory_id: entry.path ? createdDirectoryIds[entry.path] : null + try { + // ── 2. Compute checksums ── + syncing = $i18n.t('Computing checksums ({{count}} files)', { + count: pendingSyncFiles.length }); - uploadedCount++; - toast.info( - $i18n.t('Uploading: {{current}}/{{total}}', { - current: uploadedCount, - total: filesToUpload.length - }) + const manifest = await Promise.all( + pendingSyncFiles.map(async (entry) => ({ + ...entry, + checksum: await computeFileHash(entry.file), + size: entry.file.size + })) ); - } + pendingSyncFiles = null; - // ── 6. Cleanup — remove deleted files + rmdir orphaned directories ── - const staleFileIds = [ - ...diff.deleted.map((d: any) => d.file_id), - ...diff.modified.map((m: any) => m.stale_file_id) - ]; + // ── 3. Diff against knowledge base ── + syncing = $i18n.t('Comparing with knowledge base...'); + const diff = await syncKnowledgeDiff( + localStorage.token, + id, + manifest.map(({ filename, path, checksum, size }) => ({ filename, path, checksum, size })) + ); - if (staleFileIds.length > 0 || diff.rmdir.length > 0) { - await syncKnowledgeCleanup(localStorage.token, id, staleFileIds, diff.rmdir); - } + if (!diff) { + toast.error($i18n.t('Failed to compare files.')); + return; + } - // ── 7. Report ── - toast.success( - $i18n.t( - 'Sync complete: {{added}} added, {{modified}} modified, {{deleted}} deleted, {{unmodified}} unmodified', - { - added: diff.added.length, - modified: diff.modified.length, - deleted: diff.deleted.length, - unmodified: diff.unmodified_count + // ── 4. mkdir — create missing directories (parents first) ── + const createdDirectoryIds: Record = {}; + for (const dirPath of diff.mkdir) { + const segments = dirPath.split('/'); + const name = segments.at(-1)!; + const parentPath = segments.slice(0, -1).join('/'); + const parentId = parentPath ? createdDirectoryIds[parentPath] : null; + + const directory = await createKnowledgeDirectory( + localStorage.token, + knowledge.id, + name, + parentId + ); + if (directory) { + createdDirectoryIds[dirPath] = directory.id; } - ) - ); - init(); + } + + // ── 5. Upload added + modified files ── + const filesToUpload = manifest.filter( + (entry) => + diff.added.some((a: any) => a.filename === entry.filename && a.path === entry.path) || + diff.modified.some((m: any) => m.filename === entry.filename && m.path === entry.path) + ); + + let uploadedCount = 0; + for (const entry of filesToUpload) { + uploadedCount++; + const displayPath = entry.path ? `${entry.path}/${entry.filename}` : entry.filename; + syncing = $i18n.t('Uploading {{current}}/{{total}}: {{file}}', { + current: uploadedCount, + total: filesToUpload.length, + file: displayPath + }); + + const fileObject = new File([entry.file], entry.filename, { type: entry.file.type }); + await uploadFile( + localStorage.token, + fileObject, + { + knowledge_id: knowledge.id, + file_hash: entry.checksum, + directory_id: entry.path ? createdDirectoryIds[entry.path] : null + }, + null, + false + ).catch(() => null); + } + + // ── 6. Cleanup — remove deleted files + rmdir orphaned directories ── + const staleFileIds = [ + ...diff.deleted.map((d: any) => d.file_id), + ...diff.modified.map((m: any) => m.stale_file_id) + ]; + + if (staleFileIds.length > 0 || diff.rmdir.length > 0) { + syncing = $i18n.t('Removing {{count}} stale files...', { count: staleFileIds.length }); + await syncKnowledgeCleanup(localStorage.token, id, staleFileIds, diff.rmdir); + } + + // ── 7. Report ── + toast.success( + $i18n.t( + 'Sync complete: {{added}} added, {{modified}} modified, {{deleted}} deleted, {{unmodified}} unmodified', + { + added: diff.added.length, + modified: diff.modified.length, + deleted: diff.deleted.length, + unmodified: diff.unmodified_count + } + ) + ); + init(); + } catch (e) { + toast.error(`${e}`); + } finally { + syncing = null; + } }; const addFileHandler = async (fileId) => { @@ -1216,17 +1242,17 @@ /> @@ -1281,7 +1307,13 @@ {#if currentDirectoryId !== null}
- navigateToDirectory(dirId)} onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)} onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)} /> + navigateToDirectory(dirId)} + onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)} + onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)} + />
{/if} @@ -1341,6 +1373,17 @@ + {#if syncing} +
+
+ +
+ {syncing} +
+
+
+ {/if} + {#if fileItems !== null && fileItemsTotal !== null}
@@ -1372,14 +1415,14 @@ deleteFileHandler(fileId); }} onRename={(fileId, name) => renameFileHandler(fileId, name)} - onNavigateDirectory={(dirId) => navigateToDirectory(dirId)} + onNavigateDirectory={(dirId) => navigateToDirectory(dirId)} onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)} onDeleteDirectory={(id) => confirmDeleteDirectory(id)} onMoveFileToDirectory={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)} - onMoveDirectoryToDirectory={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)} - /> - + onMoveDirectoryToDirectory={(dirId, targetId) => + moveDirectoryHandler(dirId, targetId)} + />
{#if fileItemsTotal > 30}