feat: file search feature

This commit is contained in:
arvinxx
2025-12-20 22:10:38 +08:00
parent 9e47c33e9f
commit 9786d6462a
18 changed files with 2362 additions and 173 deletions
+17 -11
View File
@@ -18,13 +18,13 @@ export const HOTKEYS_REGISTRATION: HotkeyRegistration = [
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.CommandPalette,
keys: combineKeys([KeyEnum.Mod, 'j']),
keys: combineKeys([KeyEnum.Mod, 'k']),
scopes: [HotkeyScopeEnum.Global],
},
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.Search,
keys: combineKeys([KeyEnum.Mod, 'k']),
keys: combineKeys([KeyEnum.Mod, 'j']),
scopes: [HotkeyScopeEnum.Global],
},
{
@@ -40,23 +40,17 @@ export const HOTKEYS_REGISTRATION: HotkeyRegistration = [
keys: combineKeys([KeyEnum.Ctrl, KeyEnum.Backquote]),
scopes: [HotkeyScopeEnum.Global],
},
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.ToggleZenMode,
keys: combineKeys([KeyEnum.Mod, KeyEnum.Backslash]),
scopes: [HotkeyScopeEnum.Chat],
},
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.ToggleLeftPanel,
keys: combineKeys([KeyEnum.Mod, KeyEnum.BracketLeft]),
scopes: [HotkeyScopeEnum.Chat, HotkeyScopeEnum.Files, HotkeyScopeEnum.Image],
scopes: [HotkeyScopeEnum.Global],
},
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.ToggleRightPanel,
keys: combineKeys([KeyEnum.Mod, KeyEnum.BracketRight]),
scopes: [HotkeyScopeEnum.Chat, HotkeyScopeEnum.Image],
scopes: [HotkeyScopeEnum.Global],
},
{
group: HotkeyGroupEnum.Essential,
@@ -65,6 +59,12 @@ export const HOTKEYS_REGISTRATION: HotkeyRegistration = [
scopes: [HotkeyScopeEnum.Global],
},
// Chat
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.ToggleZenMode,
keys: combineKeys([KeyEnum.Mod, KeyEnum.Backslash]),
scopes: [HotkeyScopeEnum.Chat],
},
{
group: HotkeyGroupEnum.Conversation,
id: HotkeyEnum.OpenChatSettings,
@@ -99,7 +99,7 @@ export const HOTKEYS_REGISTRATION: HotkeyRegistration = [
group: HotkeyGroupEnum.Conversation,
id: HotkeyEnum.AddUserMessage,
keys: combineKeys([KeyEnum.Alt, KeyEnum.Enter]),
// Not activated through Scope mode
scopes: [HotkeyScopeEnum.Chat],
},
{
group: HotkeyGroupEnum.Conversation,
@@ -114,6 +114,12 @@ export const HOTKEYS_REGISTRATION: HotkeyRegistration = [
keys: combineKeys([KeyEnum.Alt, KeyEnum.Shift, KeyEnum.Backspace]),
scopes: [HotkeyScopeEnum.Chat],
},
{
group: HotkeyGroupEnum.Essential,
id: HotkeyEnum.SaveDocument,
keys: combineKeys([KeyEnum.Mod, 's']),
scopes: [HotkeyScopeEnum.Files],
},
];
type DesktopHotkeyRegistration = DesktopHotkeyItem[];
+6
View File
@@ -118,6 +118,12 @@ export class DocumentModel {
});
};
findBySlug = async (slug: string): Promise<DocumentItem | undefined> => {
return this.db.query.documents.findFirst({
where: and(eq(documents.userId, this.userId), eq(documents.slug, slug)),
});
};
update = async (id: string, value: Partial<DocumentItem>) => {
return this.db
.update(documents)
+5 -1
View File
@@ -40,7 +40,11 @@ export class FileModel {
}
create = async (
params: Omit<NewFile, 'id' | 'userId'> & { id?: string; knowledgeBaseId?: string },
params: Omit<NewFile, 'id' | 'userId'> & {
id?: string;
knowledgeBaseId?: string;
parentId?: string;
},
insertToGlobalFiles?: boolean,
trx?: Transaction,
): Promise<{ id: string }> => {
@@ -17,6 +17,7 @@ export interface KnowledgeItem {
metadata?: Record<string, any> | null;
name: string;
size: number;
slug?: string | null;
/**
* Source type to distinguish between files and documents
* - 'file': from files table
@@ -53,11 +54,26 @@ export class KnowledgeRepo {
sorter,
knowledgeBaseId,
showFilesInKnowledgeBase,
parentId,
limit = 50,
offset = 0,
}: QueryFileListParams = {}): Promise<KnowledgeItem[]> {
// If parentId is provided, check if it's a slug and resolve it to an ID
let resolvedParentId = parentId;
if (parentId) {
// Try to find a document with this slug
const docBySlug = await this.documentModel.findBySlug(parentId);
if (docBySlug) {
resolvedParentId = docBySlug.id;
}
// Otherwise assume it's already an ID
}
// Build file query
const fileQuery = this.buildFileQuery({
category,
knowledgeBaseId,
parentId: resolvedParentId,
q,
showFilesInKnowledgeBase,
sortType,
@@ -68,6 +84,7 @@ export class KnowledgeRepo {
const documentQuery = this.buildDocumentQuery({
category,
knowledgeBaseId,
parentId: resolvedParentId,
q,
sortType,
sorter,
@@ -80,11 +97,13 @@ export class KnowledgeRepo {
(${documentQuery})
`;
// Add final ordering
// Add final ordering and pagination
const orderClause = this.buildOrderClause(sortType, sorter);
const finalQuery = sql`
SELECT * FROM (${combinedQuery}) as combined
ORDER BY ${orderClause}
LIMIT ${limit}
OFFSET ${offset}
`;
const result = await this.db.execute(finalQuery);
@@ -123,6 +142,114 @@ export class KnowledgeRepo {
metadata,
name: row.name,
size: Number(row.size),
slug: row.slug,
sourceType: row.source_type,
updatedAt: new Date(row.updated_at),
url: row.url,
};
});
return mappedResults;
}
/**
* Query recent items (files and documents)
* Returns the most recently updated items
*/
async queryRecent(limit: number = 12): Promise<KnowledgeItem[]> {
const fileQuery = sql`
SELECT
COALESCE(d.id, f.id) as id,
f.name,
f.file_type,
f.size,
f.url,
f.created_at,
f.updated_at,
f.chunk_task_id,
f.embedding_task_id,
d.editor_data,
d.content,
d.slug,
COALESCE(d.metadata, f.metadata) as metadata,
'file' as source_type
FROM ${files} f
LEFT JOIN ${documents} d
ON f.id = d.file_id
WHERE f.user_id = ${this.userId}
AND NOT EXISTS (
SELECT 1 FROM ${knowledgeBaseFiles}
WHERE ${knowledgeBaseFiles.fileId} = f.id
)
`;
const documentQuery = sql`
SELECT
id,
COALESCE(title, filename, 'Untitled') as name,
file_type,
total_char_count as size,
source as url,
created_at,
updated_at,
NULL as chunk_task_id,
NULL as embedding_task_id,
editor_data,
content,
slug,
metadata,
'document' as source_type
FROM ${documents}
WHERE user_id = ${this.userId}
AND source_type != ${'file'}
AND (metadata->>'knowledgeBaseId') IS NULL
`;
const combinedQuery = sql`
SELECT * FROM (
(${fileQuery})
UNION ALL
(${documentQuery})
) as combined
ORDER BY updated_at DESC
LIMIT ${limit}
`;
const result = await this.db.execute(combinedQuery);
const mappedResults = result.rows.map((row: any) => {
// Parse editor_data if it's a string
let editorData = row.editor_data;
if (typeof editorData === 'string') {
try {
editorData = JSON.parse(editorData);
} catch {
editorData = null;
}
}
// Parse metadata if it's a string
let metadata = row.metadata;
if (typeof metadata === 'string') {
try {
metadata = JSON.parse(metadata);
} catch {
metadata = null;
}
}
return {
chunkTaskId: row.chunk_task_id,
content: row.content,
createdAt: new Date(row.created_at),
editorData,
embeddingTaskId: row.embedding_task_id,
fileType: row.file_type,
id: row.id,
metadata,
name: row.name,
size: Number(row.size),
slug: row.slug,
sourceType: row.source_type,
updatedAt: new Date(row.updated_at),
url: row.url,
@@ -176,25 +303,33 @@ export class KnowledgeRepo {
q,
knowledgeBaseId,
showFilesInKnowledgeBase,
parentId,
}: QueryFileListParams = {}): ReturnType<typeof sql> {
let whereConditions: any[] = [sql`${files.userId} = ${this.userId}`];
let whereConditions: any[] = [sql`f.user_id = ${this.userId}`];
// Parent ID filter
if (parentId !== undefined) {
if (parentId === null) {
whereConditions.push(sql`f.parent_id IS NULL`);
} else {
whereConditions.push(sql`f.parent_id = ${parentId}`);
}
}
// Search filter
if (q) {
whereConditions.push(sql`${files.name} ILIKE ${`%${q}%`}`);
whereConditions.push(sql`f.name ILIKE ${`%${q}%`}`);
}
// Category filter
if (category && category !== FilesTabs.All && category !== FilesTabs.Home) {
if (category && category !== FilesTabs.All) {
const fileTypePrefix = this.getFileTypePrefix(category as FilesTabs);
if (Array.isArray(fileTypePrefix)) {
// For multiple file types (e.g., Documents includes 'application' and 'custom')
const orConditions = fileTypePrefix.map(
(prefix) => sql`${files.fileType} ILIKE ${`${prefix}%`}`,
);
const orConditions = fileTypePrefix.map((prefix) => sql`f.file_type ILIKE ${`${prefix}%`}`);
whereConditions.push(sql`(${sql.join(orConditions, sql` OR `)})`);
} else {
whereConditions.push(sql`${files.fileType} ILIKE ${`${fileTypePrefix}%`}`);
whereConditions.push(sql`f.file_type ILIKE ${`${fileTypePrefix}%`}`);
}
}
@@ -203,6 +338,15 @@ export class KnowledgeRepo {
// Build where conditions using proper table references (f.column instead of files.column)
const kbWhereConditions: any[] = [sql`f.user_id = ${this.userId}`];
// Parent ID filter
if (parentId !== undefined) {
if (parentId === null) {
kbWhereConditions.push(sql`f.parent_id IS NULL`);
} else {
kbWhereConditions.push(sql`f.parent_id = ${parentId}`);
}
}
// Search filter
if (q) {
kbWhereConditions.push(sql`f.name ILIKE ${`%${q}%`}`);
@@ -223,7 +367,7 @@ export class KnowledgeRepo {
return sql`
SELECT
f.id,
COALESCE(d.id, f.id) as id,
f.name,
f.file_type,
f.size,
@@ -232,14 +376,17 @@ export class KnowledgeRepo {
f.updated_at,
f.chunk_task_id,
f.embedding_task_id,
NULL as editor_data,
NULL as content,
NULL as metadata,
d.editor_data,
d.content,
d.slug,
COALESCE(d.metadata, f.metadata) as metadata,
'file' as source_type
FROM ${files} f
INNER JOIN ${knowledgeBaseFiles} kbf
ON f.id = kbf.file_id
AND kbf.knowledge_base_id = ${knowledgeBaseId}
LEFT JOIN ${documents} d
ON f.id = d.file_id
WHERE ${sql.join(kbWhereConditions, sql` AND `)}
`;
}
@@ -250,7 +397,7 @@ export class KnowledgeRepo {
sql`
NOT EXISTS (
SELECT 1 FROM ${knowledgeBaseFiles}
WHERE ${knowledgeBaseFiles.fileId} = ${files.id}
WHERE ${knowledgeBaseFiles.fileId} = f.id
)
`,
);
@@ -258,20 +405,23 @@ export class KnowledgeRepo {
return sql`
SELECT
id,
name,
file_type,
size,
url,
created_at,
updated_at,
chunk_task_id,
embedding_task_id,
NULL as editor_data,
NULL as content,
NULL as metadata,
COALESCE(d.id, f.id) as id,
f.name,
f.file_type,
f.size,
f.url,
f.created_at,
f.updated_at,
f.chunk_task_id,
f.embedding_task_id,
d.editor_data,
d.content,
d.slug,
COALESCE(d.metadata, f.metadata) as metadata,
'file' as source_type
FROM ${files}
FROM ${files} f
LEFT JOIN ${documents} d
ON f.id = d.file_id
WHERE ${sql.join(whereConditions, sql` AND `)}
`;
}
@@ -280,12 +430,22 @@ export class KnowledgeRepo {
category,
q,
knowledgeBaseId,
parentId,
}: QueryFileListParams = {}): ReturnType<typeof sql> {
let whereConditions: any[] = [
sql`${documents.userId} = ${this.userId}`,
sql`${documents.sourceType} != ${'file'}`,
];
// Parent ID filter
if (parentId !== undefined) {
if (parentId === null) {
whereConditions.push(sql`${documents.parentId} IS NULL`);
} else {
whereConditions.push(sql`${documents.parentId} = ${parentId}`);
}
}
// Search filter
if (q) {
whereConditions.push(
@@ -294,7 +454,7 @@ export class KnowledgeRepo {
}
// Category filter - match documents by fileType prefix
if (category && category !== FilesTabs.All && category !== FilesTabs.Home) {
if (category && category !== FilesTabs.All) {
const fileTypePrefix = this.getFileTypePrefix(category as FilesTabs);
if (Array.isArray(fileTypePrefix)) {
// For multiple file types (e.g., Documents includes 'application' and 'custom')
@@ -324,6 +484,7 @@ export class KnowledgeRepo {
NULL::uuid as embedding_task_id,
NULL::jsonb as editor_data,
NULL::text as content,
NULL::varchar(255) as slug,
NULL::jsonb as metadata,
NULL::text as source_type
WHERE false
@@ -332,24 +493,92 @@ export class KnowledgeRepo {
}
// Knowledge base filter for documents
// Documents don't have knowledge base association currently, so skip if knowledgeBaseId is set
// Documents are linked to knowledge bases through files table via fileId
if (knowledgeBaseId) {
// Build where conditions using proper table references (d.column instead of documents.column)
const kbWhereConditions: any[] = [sql`d.user_id = ${this.userId}`];
// Parent ID filter
if (parentId !== undefined) {
if (parentId === null) {
kbWhereConditions.push(sql`d.parent_id IS NULL`);
} else {
kbWhereConditions.push(sql`d.parent_id = ${parentId}`);
}
}
// Search filter
if (q) {
kbWhereConditions.push(sql`(d.title ILIKE ${`%${q}%`} OR d.filename ILIKE ${`%${q}%`})`);
}
// Category filter
if (category && category !== FilesTabs.All) {
const fileTypePrefix = this.getFileTypePrefix(category as FilesTabs);
if (Array.isArray(fileTypePrefix)) {
const orConditions = fileTypePrefix.map(
(prefix) => sql`d.file_type ILIKE ${`${prefix}%`}`,
);
kbWhereConditions.push(sql`(${sql.join(orConditions, sql` OR `)})`);
// Exclude custom/document and source_type='file' from Documents category
if (category === FilesTabs.Documents) {
kbWhereConditions.push(
sql`d.file_type != ${'custom/document'}`,
sql`d.source_type != ${'file'}`,
);
}
} else if (fileTypePrefix) {
kbWhereConditions.push(sql`d.file_type ILIKE ${`${fileTypePrefix}%`}`);
} else {
// Exclude documents from other categories (Images, Videos, Audios, Websites)
return sql`
SELECT
NULL::varchar(30) as id,
NULL::text as name,
NULL::varchar(255) as file_type,
NULL::integer as size,
NULL::text as url,
NULL::timestamp with time zone as created_at,
NULL::timestamp with time zone as updated_at,
NULL::uuid as chunk_task_id,
NULL::uuid as embedding_task_id,
NULL::jsonb as editor_data,
NULL::text as content,
NULL::varchar(255) as slug,
NULL::jsonb as metadata,
NULL::text as source_type
WHERE false
`;
}
}
// When in a knowledge base, return standalone documents (folders and notes without fileId)
// that have the knowledgeBaseId set in their metadata. Documents with fileId are already
// returned by the file query via their linked file records.
kbWhereConditions.push(
sql`d.file_id IS NULL`,
sql`d.metadata->>'knowledgeBaseId' = ${knowledgeBaseId}`,
);
return sql`
SELECT
NULL::varchar(30) as id,
NULL::text as name,
NULL::varchar(255) as file_type,
NULL::integer as size,
NULL::text as url,
NULL::timestamp with time zone as created_at,
NULL::timestamp with time zone as updated_at,
NULL::uuid as chunk_task_id,
NULL::uuid as embedding_task_id,
NULL::jsonb as editor_data,
NULL::text as content,
NULL::jsonb as metadata,
NULL::text as source_type
WHERE false
d.id,
COALESCE(d.title, d.filename, 'Untitled') as name,
d.file_type,
d.total_char_count as size,
d.source as url,
d.created_at,
d.updated_at,
NULL as chunk_task_id,
NULL as embedding_task_id,
d.editor_data,
d.content,
d.slug,
d.metadata,
'document' as source_type
FROM ${documents} d
WHERE ${sql.join(kbWhereConditions, sql` AND `)}
`;
}
@@ -366,6 +595,7 @@ export class KnowledgeRepo {
NULL as embedding_task_id,
editor_data,
content,
slug,
metadata,
'document' as source_type
FROM ${documents}
@@ -0,0 +1,759 @@
// @vitest-environment node
import { beforeEach, describe, expect, it } from 'vitest';
import { getTestDB } from '../../models/__tests__/_util';
import { NewAgent, agents } from '../../schemas/agent';
import { NewFile, files } from '../../schemas/file';
import { messages } from '../../schemas/message';
import { NewTopic, topics } from '../../schemas/topic';
import { users } from '../../schemas/user';
import { LobeChatDatabase } from '../../type';
import { SearchRepo } from './index';
const userId = 'search-test-user';
const otherUserId = 'other-search-user';
let searchRepo: SearchRepo;
const serverDB: LobeChatDatabase = await getTestDB();
beforeEach(async () => {
// Clean up
await serverDB.delete(users);
// Create test users
await serverDB.insert(users).values([{ id: userId }, { id: otherUserId }]);
// Initialize repo
searchRepo = new SearchRepo(serverDB, userId);
});
describe('SearchRepo', () => {
describe('search - empty query', () => {
it('should return empty array for empty query', async () => {
const results = await searchRepo.search({ query: '' });
expect(results).toEqual([]);
});
it('should return empty array for whitespace query', async () => {
const results = await searchRepo.search({ query: ' ' });
expect(results).toEqual([]);
});
});
describe('search - basic search', () => {
beforeEach(async () => {
// Create test agents
const testAgents: NewAgent[] = [
{
description: 'A helpful React coding assistant',
slug: 'react-helper',
tags: ['react', 'frontend', 'coding'],
title: 'React Helper',
userId,
},
{
description: 'Python development assistant',
slug: 'python-dev',
tags: ['python', 'backend'],
title: 'Python Developer',
userId,
},
];
await serverDB.insert(agents).values(testAgents);
// Create test topics
const testTopics: NewTopic[] = [
{
content: 'Discussion about React hooks and best practices',
title: 'React Hooks Guide',
userId,
},
{
content: 'Notes on Python async programming',
title: 'Python Async Notes',
userId,
},
];
await serverDB.insert(topics).values(testTopics);
// Create test files
const testFiles: NewFile[] = [
{
fileType: 'application/javascript',
name: 'react-component.jsx',
size: 1024,
url: 'file://react-component.jsx',
userId,
},
{
fileType: 'text/python',
name: 'python-script.py',
size: 2048,
url: 'file://python-script.py',
userId,
},
];
await serverDB.insert(files).values(testFiles);
});
it('should find agents by title', async () => {
const results = await searchRepo.search({ query: 'React Helper' });
const agentResults = results.filter((r) => r.type === 'agent');
expect(agentResults).toHaveLength(1);
expect(agentResults[0].title).toBe('React Helper');
});
it('should find topics by title', async () => {
const results = await searchRepo.search({ query: 'React Hooks' });
const topicResults = results.filter((r) => r.type === 'topic');
expect(topicResults).toHaveLength(1);
expect(topicResults[0].title).toBe('React Hooks Guide');
});
it('should find files by name', async () => {
const results = await searchRepo.search({ query: 'react-component' });
const fileResults = results.filter((r) => r.type === 'file');
expect(fileResults).toHaveLength(1);
expect(fileResults[0].title).toBe('react-component.jsx');
});
it('should find results across all types', async () => {
const results = await searchRepo.search({ query: 'react' });
// Should find: 1 agent, 1 topic, 1 file
expect(results.length).toBeGreaterThanOrEqual(3);
const types = new Set(results.map((r) => r.type));
expect(types.has('agent')).toBe(true);
expect(types.has('topic')).toBe(true);
expect(types.has('file')).toBe(true);
});
it('should search in agent description', async () => {
const results = await searchRepo.search({ query: 'coding assistant' });
const agentResults = results.filter((r) => r.type === 'agent');
expect(agentResults.length).toBeGreaterThanOrEqual(1);
expect(agentResults[0].description).toContain('coding');
});
it('should search in topic content', async () => {
const results = await searchRepo.search({ query: 'async programming' });
const topicResults = results.filter((r) => r.type === 'topic');
expect(topicResults.length).toBeGreaterThanOrEqual(1);
expect(topicResults[0].description).toContain('async');
});
it('should search in agent tags', async () => {
const results = await searchRepo.search({ query: 'frontend' });
const agentResults = results.filter((r) => r.type === 'agent');
expect(agentResults.length).toBeGreaterThanOrEqual(1);
expect(agentResults[0].tags).toContain('frontend');
});
});
describe('search - relevance ranking', () => {
beforeEach(async () => {
const testAgents: NewAgent[] = [
{
slug: 'exact',
title: 'test',
userId,
},
{
slug: 'prefix',
title: 'testing',
userId,
},
{
slug: 'contains',
title: 'my test agent',
userId,
},
];
await serverDB.insert(agents).values(testAgents);
});
it('should prioritize exact match (relevance=1)', async () => {
const results = await searchRepo.search({ query: 'test' });
const exactMatch = results.find((r) => r.type === 'agent' && r.slug === 'exact');
expect(exactMatch).toBeDefined();
expect(exactMatch?.relevance).toBe(1);
});
it('should rank prefix match second (relevance=2)', async () => {
const results = await searchRepo.search({ query: 'test' });
const prefixMatch = results.find((r) => r.type === 'agent' && r.slug === 'prefix');
expect(prefixMatch).toBeDefined();
expect(prefixMatch?.relevance).toBe(2);
});
it('should rank contains match third (relevance=3)', async () => {
const results = await searchRepo.search({ query: 'test' });
const containsMatch = results.find((r) => r.type === 'agent' && r.slug === 'contains');
expect(containsMatch).toBeDefined();
expect(containsMatch?.relevance).toBe(3);
});
it('should order results by relevance', async () => {
const results = await searchRepo.search({ query: 'test' });
const agentResults = results.filter((r) => r.type === 'agent');
// Exact match should come first
expect(agentResults[0].slug).toBe('exact');
expect(agentResults[0].relevance).toBe(1);
// Prefix match should come second
expect(agentResults[1].slug).toBe('prefix');
expect(agentResults[1].relevance).toBe(2);
// Contains match should come third
expect(agentResults[2].slug).toBe('contains');
expect(agentResults[2].relevance).toBe(3);
});
});
describe('search - user isolation', () => {
beforeEach(async () => {
// Create agent for current user
await serverDB.insert(agents).values({
slug: 'user-agent',
title: 'User Agent',
userId,
});
// Create agent for other user
await serverDB.insert(agents).values({
slug: 'other-agent',
title: 'Other Agent',
userId: otherUserId,
});
// Create topic for current user
await serverDB.insert(topics).values({
title: 'User Topic',
userId,
});
// Create topic for other user
await serverDB.insert(topics).values({
title: 'Other Topic',
userId: otherUserId,
});
// Create file for current user
await serverDB.insert(files).values({
fileType: 'text/plain',
name: 'user-file.txt',
size: 100,
url: 'file://user-file.txt',
userId,
});
// Create file for other user
await serverDB.insert(files).values({
fileType: 'text/plain',
name: 'other-file.txt',
size: 100,
url: 'file://other-file.txt',
userId: otherUserId,
});
});
it('should only return current user results', async () => {
const results = await searchRepo.search({ query: 'agent' });
expect(results.length).toBeGreaterThan(0);
// All results should be from current user
results.forEach((result) => {
expect(result.title).not.toContain('Other');
});
});
it('should not return other user agents', async () => {
const results = await searchRepo.search({ query: 'agent' });
const otherAgent = results.find((r) => r.title === 'Other Agent');
expect(otherAgent).toBeUndefined();
});
it('should not return other user topics', async () => {
const results = await searchRepo.search({ query: 'topic' });
const otherTopic = results.find((r) => r.title === 'Other Topic');
expect(otherTopic).toBeUndefined();
});
it('should not return other user files', async () => {
const results = await searchRepo.search({ query: 'file' });
const otherFile = results.find((r) => r.title === 'other-file.txt');
expect(otherFile).toBeUndefined();
});
});
describe('search - type filtering', () => {
beforeEach(async () => {
await serverDB.insert(agents).values({
slug: 'test-agent',
title: 'Test Agent',
userId,
});
await serverDB.insert(topics).values({
title: 'Test Topic',
userId,
});
await serverDB.insert(files).values({
fileType: 'text/plain',
name: 'test-file.txt',
size: 100,
url: 'file://test-file.txt',
userId,
});
});
it('should filter by agent type', async () => {
const results = await searchRepo.search({ query: 'test', type: 'agent' });
expect(results.length).toBeGreaterThan(0);
results.forEach((result) => {
expect(result.type).toBe('agent');
});
});
it('should filter by topic type', async () => {
const results = await searchRepo.search({ query: 'test', type: 'topic' });
expect(results.length).toBeGreaterThan(0);
results.forEach((result) => {
expect(result.type).toBe('topic');
});
});
it('should filter by file type', async () => {
const results = await searchRepo.search({ query: 'test', type: 'file' });
expect(results.length).toBeGreaterThan(0);
results.forEach((result) => {
expect(result.type).toBe('file');
});
});
});
describe('search - limit per type', () => {
beforeEach(async () => {
// Create 10 agents
const testAgents: NewAgent[] = Array.from({ length: 10 }, (_, i) => ({
slug: `agent-${i}`,
title: `Test Agent ${i}`,
userId,
}));
await serverDB.insert(agents).values(testAgents);
});
it('should respect default limit of 5 per type', async () => {
const results = await searchRepo.search({ query: 'test' });
const agentResults = results.filter((r) => r.type === 'agent');
expect(agentResults.length).toBeLessThanOrEqual(5);
});
it('should respect custom limit per type', async () => {
const results = await searchRepo.search({ limitPerType: 3, query: 'test' });
const agentResults = results.filter((r) => r.type === 'agent');
expect(agentResults.length).toBeLessThanOrEqual(3);
});
});
describe('search - case insensitivity', () => {
beforeEach(async () => {
await serverDB.insert(agents).values({
description: 'React Development Assistant',
slug: 'react-agent',
title: 'React Agent',
userId,
});
});
it('should search case-insensitively', async () => {
const upperResults = await searchRepo.search({ query: 'REACT' });
const lowerResults = await searchRepo.search({ query: 'react' });
const mixedResults = await searchRepo.search({ query: 'ReAcT' });
expect(upperResults.length).toBeGreaterThan(0);
expect(lowerResults.length).toBeGreaterThan(0);
expect(mixedResults.length).toBeGreaterThan(0);
// All should return the same agent
expect(upperResults[0].id).toBe(lowerResults[0].id);
expect(lowerResults[0].id).toBe(mixedResults[0].id);
});
});
describe('search - result structure', () => {
beforeEach(async () => {
await serverDB.insert(agents).values({
avatar: 'avatar-url',
backgroundColor: '#ff0000',
description: 'Test description',
slug: 'test-agent',
tags: ['tag1', 'tag2'],
title: 'Test Agent',
userId,
});
await serverDB.insert(topics).values({
content: 'Test content',
favorite: true,
title: 'Test Topic',
userId,
});
await serverDB.insert(files).values({
fileType: 'text/plain',
name: 'test.txt',
size: 100,
url: 'file://test.txt',
userId,
});
});
it('should return correct agent result structure', async () => {
const results = await searchRepo.search({ query: 'test', type: 'agent' });
expect(results.length).toBeGreaterThan(0);
const agent = results[0];
expect(agent.type).toBe('agent');
expect(agent.id).toBeDefined();
expect(agent.title).toBeDefined();
expect(agent.relevance).toBeGreaterThan(0);
expect(agent.createdAt).toBeInstanceOf(Date);
expect(agent.updatedAt).toBeInstanceOf(Date);
if (agent.type === 'agent') {
expect(agent.slug).toBeDefined();
expect(agent.tags).toBeInstanceOf(Array);
}
});
it('should return correct topic result structure', async () => {
const results = await searchRepo.search({ query: 'test', type: 'topic' });
expect(results.length).toBeGreaterThan(0);
const topic = results[0];
expect(topic.type).toBe('topic');
expect(topic.id).toBeDefined();
expect(topic.title).toBeDefined();
expect(topic.relevance).toBeGreaterThan(0);
expect(topic.createdAt).toBeInstanceOf(Date);
expect(topic.updatedAt).toBeInstanceOf(Date);
if (topic.type === 'topic') {
expect(topic.favorite).toBeDefined();
}
});
it('should return correct file result structure', async () => {
const results = await searchRepo.search({ query: 'test', type: 'file' });
expect(results.length).toBeGreaterThan(0);
const file = results[0];
expect(file.type).toBe('file');
expect(file.id).toBeDefined();
expect(file.title).toBeDefined();
expect(file.relevance).toBeGreaterThan(0);
expect(file.createdAt).toBeInstanceOf(Date);
expect(file.updatedAt).toBeInstanceOf(Date);
if (file.type === 'file') {
expect(file.name).toBeDefined();
expect(file.fileType).toBeDefined();
expect(file.size).toBeGreaterThanOrEqual(0);
}
});
});
describe('search - agent context awareness', () => {
let testAgentId: string;
let otherAgentId: string;
beforeEach(async () => {
// Create test agents
const [agent1, agent2] = await serverDB
.insert(agents)
.values([
{
slug: 'test-agent',
title: 'Test Agent',
userId,
},
{
slug: 'other-agent',
title: 'Other Agent',
userId,
},
])
.returning();
testAgentId = agent1.id;
otherAgentId = agent2.id;
// Create topics for test agent
await serverDB.insert(topics).values([
{
agentId: testAgentId,
title: 'React Testing Guide',
userId,
},
{
agentId: testAgentId,
title: 'Testing Best Practices',
userId,
},
]);
// Create topics for other agent
await serverDB.insert(topics).values([
{
agentId: otherAgentId,
title: 'Testing Strategies',
userId,
},
]);
// Create topic without agent
await serverDB.insert(topics).values([
{
agentId: null,
title: 'General Testing Tips',
userId,
},
]);
});
it('should boost current agent topics in relevance', async () => {
const results = await searchRepo.search({
agentId: testAgentId,
query: 'testing',
});
const topicResults = results.filter((r) => r.type === 'topic');
// Current agent's topics should have better relevance (0.5-0.7)
const currentAgentTopics = topicResults.filter(
(t) => t.type === 'topic' && t.agentId === testAgentId,
);
const otherTopics = topicResults.filter(
(t) => t.type === 'topic' && t.agentId !== testAgentId,
);
expect(currentAgentTopics.length).toBeGreaterThan(0);
expect(otherTopics.length).toBeGreaterThan(0);
// Current agent topics should have lower relevance scores (higher priority)
currentAgentTopics.forEach((topic) => {
expect(topic.relevance).toBeLessThan(1);
});
otherTopics.forEach((topic) => {
expect(topic.relevance).toBeGreaterThanOrEqual(1);
});
});
it('should show all user topics but rank current agent topics first', async () => {
const results = await searchRepo.search({
agentId: testAgentId,
query: 'testing',
});
const topicResults = results.filter((r) => r.type === 'topic');
// Should include topics from all agents (current, other, and no agent)
const agentIds = new Set(topicResults.map((t) => (t.type === 'topic' ? t.agentId : null)));
expect(agentIds.has(testAgentId)).toBe(true);
expect(agentIds.has(otherAgentId)).toBe(true);
expect(agentIds.has(null)).toBe(true);
// First results should be from current agent
expect(topicResults[0].type).toBe('topic');
if (topicResults[0].type === 'topic') {
expect(topicResults[0].agentId).toBe(testAgentId);
}
});
it('should return 6 topics in agent context', async () => {
// Create additional topics to test limit
await serverDB.insert(topics).values(
Array.from({ length: 12 }, (_, i) => ({
agentId: i < 6 ? testAgentId : otherAgentId,
title: `Test Topic ${i}`,
userId,
})),
);
const results = await searchRepo.search({
agentId: testAgentId,
query: 'test',
});
const topicResults = results.filter((r) => r.type === 'topic');
expect(topicResults.length).toBe(6);
});
it('should return 3 agents and 3 files in agent context', async () => {
// Create test agents
await serverDB.insert(agents).values(
Array.from({ length: 5 }, (_, i) => ({
slug: `agent-${i}`,
title: `Test Agent ${i}`,
userId,
})),
);
// Create test files
await serverDB.insert(files).values(
Array.from({ length: 5 }, (_, i) => ({
fileType: 'text/plain',
name: `test-file-${i}.txt`,
size: 100,
url: `file://test-file-${i}.txt`,
userId,
})),
);
const results = await searchRepo.search({
agentId: testAgentId,
query: 'test',
});
const agentResults = results.filter((r) => r.type === 'agent');
const fileResults = results.filter((r) => r.type === 'file');
expect(agentResults.length).toBeLessThanOrEqual(3);
expect(fileResults.length).toBeLessThanOrEqual(3);
});
it('should use normal limits without agent context', async () => {
const results = await searchRepo.search({
query: 'testing',
});
const topicResults = results.filter((r) => r.type === 'topic');
// Should use default limit of 3 per type
expect(topicResults.length).toBeLessThanOrEqual(3);
});
it('should not boost topics when agentId is not provided', async () => {
const results = await searchRepo.search({
query: 'testing',
});
const topicResults = results.filter((r) => r.type === 'topic');
// All topics should have normal relevance (1-3)
topicResults.forEach((topic) => {
expect(topic.relevance).toBeGreaterThanOrEqual(1);
expect(topic.relevance).toBeLessThanOrEqual(3);
});
});
});
describe('search - message search', () => {
beforeEach(async () => {
// Create test messages with different roles
await serverDB.insert(messages).values([
{
content: 'Hello, I need help with React hooks',
role: 'user',
userId,
},
{
content: 'Sure, I can help you with React hooks and state management',
role: 'assistant',
userId,
},
{
content: 'Tool call result for React documentation lookup',
role: 'tool',
userId,
},
{
content: 'Another tool message about hooks',
role: 'tool',
userId,
},
]);
});
it('should find messages by content', async () => {
const results = await searchRepo.search({ query: 'React hooks', type: 'message' });
expect(results.length).toBeGreaterThan(0);
results.forEach((result) => {
expect(result.type).toBe('message');
});
});
it('should filter out messages with role=tool', async () => {
const results = await searchRepo.search({ query: 'tool', type: 'message' });
// Should not find any tool messages even though they contain "tool" in content
const toolMessages = results.filter((r) => r.type === 'message' && r.role === 'tool');
expect(toolMessages.length).toBe(0);
});
it('should return user and assistant messages but not tool messages', async () => {
const results = await searchRepo.search({ query: 'hooks' });
const messageResults = results.filter((r) => r.type === 'message');
// Should find user and assistant messages
expect(messageResults.length).toBeGreaterThan(0);
// Verify all returned messages are not tool messages
messageResults.forEach((msg) => {
if (msg.type === 'message') {
expect(msg.role).not.toBe('tool');
}
});
});
it('should return correct message structure', async () => {
const results = await searchRepo.search({ query: 'help', type: 'message' });
expect(results.length).toBeGreaterThan(0);
const message = results[0];
expect(message.type).toBe('message');
expect(message.id).toBeDefined();
expect(message.title).toBeDefined();
expect(message.relevance).toBeGreaterThan(0);
expect(message.createdAt).toBeInstanceOf(Date);
expect(message.updatedAt).toBeInstanceOf(Date);
if (message.type === 'message') {
expect(message.content).toBeDefined();
expect(message.role).toBeDefined();
expect(['user', 'assistant']).toContain(message.role);
}
});
});
});
@@ -0,0 +1,756 @@
import { sql } from 'drizzle-orm';
import { agents, documents, files, knowledgeBaseFiles, messages, topics } from '../../schemas';
import { LobeChatDatabase } from '../../type';
export type SearchResultType =
| 'page'
| 'pageContent'
| 'agent'
| 'topic'
| 'file'
| 'message'
| 'mcp'
| 'plugin'
| 'assistant';
export interface BaseSearchResult {
// 1=exact, 2=prefix, 3=contains
createdAt: Date;
description?: string | null;
id: string;
relevance: number;
title: string;
type: SearchResultType;
updatedAt: Date;
}
export interface PageSearchResult extends BaseSearchResult {
id: string;
type: 'page';
}
export interface PageContentSearchResult extends BaseSearchResult {
id: string;
type: 'pageContent';
}
export interface AgentSearchResult extends BaseSearchResult {
avatar: string | null;
backgroundColor: string | null;
slug: string | null;
tags: string[];
type: 'agent';
}
export interface TopicSearchResult extends BaseSearchResult {
agentId: string | null;
favorite: boolean | null;
sessionId: string | null;
type: 'topic';
}
export interface FileSearchResult extends BaseSearchResult {
fileType: string;
knowledgeBaseId: string | null;
name: string;
size: number;
type: 'file';
url: string | null;
}
export interface MessageSearchResult extends BaseSearchResult {
agentId: string | null;
content: string;
model: string | null;
role: string;
topicId: string | null;
type: 'message';
}
export interface MCPSearchResult extends BaseSearchResult {
author: string;
avatar?: string | null;
category?: string | null;
connectionType?: 'http' | 'stdio' | null;
identifier: string;
installCount?: number | null;
isFeatured?: boolean | null;
isValidated?: boolean | null;
tags?: string[] | null;
type: 'mcp';
}
export interface PluginSearchResult extends BaseSearchResult {
author: string;
avatar?: string | null;
category?: string | null;
identifier: string;
tags?: string[] | null;
type: 'plugin';
}
export interface AssistantSearchResult extends BaseSearchResult {
author: string;
avatar?: string | null;
homepage?: string | null;
identifier: string;
tags?: string[] | null;
type: 'assistant';
}
export type SearchResult =
| PageSearchResult
| PageContentSearchResult
| AgentSearchResult
| TopicSearchResult
| FileSearchResult
| MessageSearchResult
| MCPSearchResult
| PluginSearchResult
| AssistantSearchResult;
export interface SearchOptions {
agentId?: string;
contextType?: 'agent' | 'resource' | 'page';
limitPerType?: number;
offset?: number;
query: string;
type?: SearchResultType;
}
/**
* Search Repository - provides unified search across Agents, Topics, and Files
*/
export class SearchRepo {
private userId: string;
private db: LobeChatDatabase;
constructor(db: LobeChatDatabase, userId: string) {
this.userId = userId;
this.db = db;
}
/**
* Search across agents, topics, files, and pages
*/
async search(options: SearchOptions): Promise<SearchResult[]> {
const { query, type, limitPerType = 5, agentId, contextType } = options;
// Early return for empty query
if (!query || query.trim() === '') return [];
const trimmedQuery = query.trim();
const searchTerm = `%${trimmedQuery}%`;
const exactQuery = trimmedQuery;
const prefixQuery = `${trimmedQuery}%`;
// Context-aware limits: prioritize relevant types based on context
const limits = this.calculateLimits(limitPerType, type, agentId, contextType);
// Build queries based on type filter
const queries = [];
if (!type || type === 'agent') {
queries.push(this.buildAgentQuery(searchTerm, exactQuery, prefixQuery, limits.agent));
}
if (!type || type === 'topic') {
queries.push(
this.buildTopicQuery(searchTerm, exactQuery, prefixQuery, limits.topic, agentId),
);
}
if (!type || type === 'message') {
queries.push(
this.buildMessageQuery(searchTerm, exactQuery, prefixQuery, limits.message, agentId),
);
}
if (!type || type === 'file') {
queries.push(this.buildFileQuery(searchTerm, exactQuery, prefixQuery, limits.file));
}
if (!type || type === 'page') {
queries.push(this.buildPageQuery(searchTerm, exactQuery, prefixQuery, limits.page));
}
if (queries.length === 0) return [];
// Combine with UNION ALL (pattern from KnowledgeRepo)
const unionQuery = sql.join(
queries.map((q) => sql`(${q})`),
sql` UNION ALL `,
);
const finalQuery = sql`
SELECT * FROM (${unionQuery}) as combined
ORDER BY relevance ASC, updated_at DESC
`;
const result = await this.db.execute(finalQuery);
return this.mapResults(result.rows as any[]);
}
/**
* Calculate result limits based on context
* - Agent context: expand topics (6) and messages (6), limit others (3 each)
* - Page context: expand pages (6), limit others (3 each)
* - Resource context: expand files (6), limit others (3 each)
* - General context: limit all types to 3 each
*/
private calculateLimits(
baseLimit: number,
type?: SearchResultType,
agentId?: string,
contextType?: 'agent' | 'resource' | 'page',
): {
agent: number;
file: number;
message: number;
page: number;
pageContent: number;
topic: number;
} {
// If type filter is specified, use full limit for that type
if (type) {
return {
agent: type === 'agent' ? baseLimit : 0,
file: type === 'file' ? baseLimit : 0,
message: type === 'message' ? baseLimit : 0,
page: type === 'page' ? baseLimit : 0,
pageContent: type === 'pageContent' ? baseLimit : 0,
topic: type === 'topic' ? baseLimit : 0,
};
}
// Page context: expand pages to 6, limit others to 3
if (contextType === 'page') {
return {
agent: 3,
file: 3,
message: 3,
page: 6,
pageContent: 0, // Not available yet
topic: 3,
};
}
// Resource context: expand files to 6, limit others to 3
if (contextType === 'resource') {
return {
agent: 3,
file: 6,
message: 3,
page: 3,
pageContent: 0, // Not available yet
topic: 3,
};
}
// Agent context: expand topics and messages to 6, limit others to 3
if (agentId || contextType === 'agent') {
return {
agent: 3,
file: 3,
message: 6,
page: 3,
pageContent: 0, // Not available yet
topic: 6,
};
}
// General context: limit all types to 3
return {
agent: 3,
file: 3,
message: 3,
page: 3,
pageContent: 0, // Not available yet
topic: 3,
};
}
/**
* Build agent search query
* Searches: title, description, slug, tags (JSONB array)
*/
private buildAgentQuery(
searchTerm: string,
exactQuery: string,
prefixQuery: string,
limit: number,
): ReturnType<typeof sql> {
return sql`
SELECT
a.id,
'agent' as type,
a.title,
a.description,
a.slug,
a.avatar,
a.background_color,
a.tags,
a.created_at,
a.updated_at,
CASE
WHEN a.title ILIKE ${exactQuery} THEN 1
WHEN a.title ILIKE ${prefixQuery} THEN 2
ELSE 3
END as relevance,
NULL::boolean as favorite,
NULL::text as session_id,
NULL::text as agent_id,
NULL::text as name,
NULL::varchar(255) as file_type,
NULL::integer as size,
NULL::text as url,
NULL::text as knowledge_base_id
FROM ${agents} a
WHERE a.user_id = ${this.userId}
AND (
a.title ILIKE ${searchTerm}
OR COALESCE(a.description, '') ILIKE ${searchTerm}
OR COALESCE(a.slug, '') ILIKE ${searchTerm}
OR (
a.tags IS NOT NULL
AND EXISTS (
SELECT 1 FROM jsonb_array_elements_text(a.tags) AS tag
WHERE tag ILIKE ${searchTerm}
)
)
)
LIMIT ${limit}
`;
}
/**
* Build topic search query with optional agent-context boosting
* Searches: title, content, historySummary
* When agentId is provided:
* - Current agent's topics: relevance 0.5-0.7 (highest priority)
* - Other topics: relevance 1-3 (normal priority)
*/
private buildTopicQuery(
searchTerm: string,
exactQuery: string,
prefixQuery: string,
limit: number,
agentId?: string,
): ReturnType<typeof sql> {
// Build relevance CASE statement with agent boosting
const relevanceCase = agentId
? sql`
CASE
WHEN t.agent_id = ${agentId} THEN
CASE
WHEN t.title ILIKE ${exactQuery} THEN 0.5
WHEN t.title ILIKE ${prefixQuery} THEN 0.6
ELSE 0.7
END
WHEN t.title ILIKE ${exactQuery} THEN 1
WHEN t.title ILIKE ${prefixQuery} THEN 2
ELSE 3
END
`
: sql`
CASE
WHEN t.title ILIKE ${exactQuery} THEN 1
WHEN t.title ILIKE ${prefixQuery} THEN 2
ELSE 3
END
`;
return sql`
SELECT
t.id,
'topic' as type,
t.title,
t.content as description,
NULL::varchar(100) as slug,
NULL::text as avatar,
NULL::text as background_color,
NULL::jsonb as tags,
t.created_at,
t.updated_at,
${relevanceCase} as relevance,
t.favorite,
t.session_id,
t.agent_id,
NULL::text as name,
NULL::varchar(255) as file_type,
NULL::integer as size,
NULL::text as url,
NULL::text as knowledge_base_id
FROM ${topics} t
WHERE t.user_id = ${this.userId}
AND (
COALESCE(t.title, '') ILIKE ${searchTerm}
OR COALESCE(t.content, '') ILIKE ${searchTerm}
OR COALESCE(t.history_summary, '') ILIKE ${searchTerm}
)
ORDER BY relevance ASC, t.updated_at DESC
LIMIT ${limit}
`;
}
/**
* Build message search query with optional agent-context boosting
* Searches: message content (supports multi-word queries)
* When agentId is provided:
* - Current agent's messages: relevance 0.5-0.7 (highest priority)
* - Other messages: relevance 1-3 (normal priority)
*/
private buildMessageQuery(
searchTerm: string,
exactQuery: string,
prefixQuery: string,
limit: number,
agentId?: string,
): ReturnType<typeof sql> {
// Split search query into words for better multi-word search
const words = exactQuery
.trim()
.split(/\s+/)
.filter((w) => w.length > 0);
// Build WHERE clause: search for any of the words
const wordConditions =
words.length > 1
? sql.join(
words.map((word) => sql`COALESCE(m.content, '') ILIKE ${`%${word}%`}`),
sql` OR `,
)
: sql`COALESCE(m.content, '') ILIKE ${searchTerm}`;
// Build relevance CASE statement with agent boosting
const relevanceCase = agentId
? sql`
CASE
WHEN m.agent_id = ${agentId} THEN
CASE
WHEN m.content ILIKE ${exactQuery} THEN 0.5
WHEN m.content ILIKE ${prefixQuery} THEN 0.6
ELSE 0.7
END
WHEN m.content ILIKE ${exactQuery} THEN 1
WHEN m.content ILIKE ${prefixQuery} THEN 2
ELSE 3
END
`
: sql`
CASE
WHEN m.content ILIKE ${exactQuery} THEN 1
WHEN m.content ILIKE ${prefixQuery} THEN 2
ELSE 3
END
`;
return sql`
SELECT
m.id,
'message' as type,
CASE
WHEN length(m.content) > 100 THEN substring(m.content, 1, 100) || '...'
ELSE m.content
END as title,
COALESCE(a.title, 'General Chat') as description,
m.model as slug,
NULL::text as avatar,
NULL::text as background_color,
NULL::jsonb as tags,
m.created_at,
m.updated_at,
${relevanceCase} as relevance,
NULL::boolean as favorite,
m.topic_id as session_id,
m.agent_id,
m.role as name,
NULL::varchar(255) as file_type,
NULL::integer as size,
NULL::text as url,
NULL::text as knowledge_base_id
FROM ${messages} m
LEFT JOIN ${agents} a ON m.agent_id = a.id
WHERE m.user_id = ${this.userId}
AND m.role != 'tool'
AND (${wordConditions})
ORDER BY relevance ASC, m.created_at DESC
LIMIT ${limit}
`;
}
/**
* Build file search query
* Searches files and their linked documents, excluding pages (file_type='custom/document')
*/
private buildFileQuery(
searchTerm: string,
exactQuery: string,
prefixQuery: string,
limit: number,
): ReturnType<typeof sql> {
// Query for files (with optional linked documents), excluding custom/document files
const fileQuery = sql`
SELECT
f.id,
'file' as type,
f.name as title,
d.content as description,
NULL::varchar(100) as slug,
NULL::text as avatar,
NULL::text as background_color,
NULL::jsonb as tags,
f.created_at,
f.updated_at,
CASE
WHEN f.name ILIKE ${exactQuery} THEN 1
WHEN f.name ILIKE ${prefixQuery} THEN 2
ELSE 3
END as relevance,
NULL::boolean as favorite,
NULL::text as session_id,
NULL::text as agent_id,
f.name,
f.file_type,
f.size,
f.url,
kbf.knowledge_base_id
FROM ${files} f
LEFT JOIN ${documents} d ON f.id = d.file_id
LEFT JOIN ${knowledgeBaseFiles} kbf ON f.id = kbf.file_id
WHERE f.user_id = ${this.userId}
AND f.file_type != 'custom/document'
AND f.name ILIKE ${searchTerm}
`;
// Query for standalone documents (not pages and not linked to files)
const documentQuery = sql`
SELECT
d.id,
'file' as type,
COALESCE(d.title, d.filename, 'Untitled') as title,
d.content as description,
NULL::varchar(100) as slug,
NULL::text as avatar,
NULL::text as background_color,
NULL::jsonb as tags,
d.created_at,
d.updated_at,
CASE
WHEN COALESCE(d.title, d.filename) ILIKE ${exactQuery} THEN 1
WHEN COALESCE(d.title, d.filename) ILIKE ${prefixQuery} THEN 2
ELSE 3
END as relevance,
NULL::boolean as favorite,
NULL::text as session_id,
NULL::text as agent_id,
COALESCE(d.title, d.filename, 'Untitled') as name,
d.file_type,
d.total_char_count as size,
d.source as url,
kbf.knowledge_base_id
FROM ${documents} d
LEFT JOIN ${files} f ON d.file_id = f.id
LEFT JOIN ${knowledgeBaseFiles} kbf ON f.id = kbf.file_id
WHERE d.user_id = ${this.userId}
AND d.source_type != 'file'
AND d.file_type != 'custom/document'
AND (
COALESCE(d.title, '') ILIKE ${searchTerm}
OR COALESCE(d.filename, '') ILIKE ${searchTerm}
OR COALESCE(d.content, '') ILIKE ${searchTerm}
)
`;
// Combine both queries
return sql`
SELECT * FROM (
(${fileQuery})
UNION ALL
(${documentQuery})
) as combined
ORDER BY relevance ASC, updated_at DESC
LIMIT ${limit}
`;
}
/**
* Build page search query
* Fast search on page titles only (no content search for better performance)
* Searches standalone documents with type='custom/document'
*/
private buildPageQuery(
searchTerm: string,
exactQuery: string,
prefixQuery: string,
limit: number,
): ReturnType<typeof sql> {
return sql`
SELECT
d.id,
'page' as type,
COALESCE(d.title, d.filename, 'Untitled') as title,
NULL::text as description,
NULL::varchar(100) as slug,
NULL::text as avatar,
NULL::text as background_color,
NULL::jsonb as tags,
d.created_at,
d.updated_at,
CASE
WHEN COALESCE(d.title, d.filename) ILIKE ${exactQuery} THEN 1
WHEN COALESCE(d.title, d.filename) ILIKE ${prefixQuery} THEN 2
ELSE 3
END as relevance,
NULL::boolean as favorite,
NULL::text as session_id,
NULL::text as agent_id,
COALESCE(d.title, d.filename, 'Untitled') as name,
d.file_type,
d.total_char_count as size,
d.source as url,
NULL::text as knowledge_base_id
FROM ${documents} d
WHERE d.user_id = ${this.userId}
AND d.file_type = 'custom/document'
AND (
COALESCE(d.title, '') ILIKE ${searchTerm}
OR COALESCE(d.filename, '') ILIKE ${searchTerm}
)
ORDER BY relevance ASC, updated_at DESC
LIMIT ${limit}
`;
}
/**
* Build page content search query (FUTURE USE - Not integrated yet)
* Full-text search within page content for deep document search
* This is more expensive but allows searching within document body
*/
private buildPageContentQuery(
searchTerm: string,
exactQuery: string,
prefixQuery: string,
limit: number,
): ReturnType<typeof sql> {
return sql`
SELECT
d.id,
'pageContent' as type,
COALESCE(d.title, d.filename, 'Untitled') as title,
d.content as description,
NULL::varchar(100) as slug,
NULL::text as avatar,
NULL::text as background_color,
NULL::jsonb as tags,
d.created_at,
d.updated_at,
CASE
WHEN COALESCE(d.content, '') ILIKE ${exactQuery} THEN 1
WHEN COALESCE(d.content, '') ILIKE ${prefixQuery} THEN 2
ELSE 3
END as relevance,
NULL::boolean as favorite,
NULL::text as session_id,
NULL::text as agent_id,
COALESCE(d.title, d.filename, 'Untitled') as name,
d.file_type,
d.total_char_count as size,
d.source as url,
NULL::text as knowledge_base_id
FROM ${documents} d
WHERE d.user_id = ${this.userId}
AND d.file_type = 'custom/document'
AND COALESCE(d.content, '') ILIKE ${searchTerm}
ORDER BY relevance ASC, updated_at DESC
LIMIT ${limit}
`;
}
/**
* Map raw SQL results to typed SearchResult objects
* Parse JSONB strings and convert snake_case to camelCase
*/
private mapResults(rows: any[]): SearchResult[] {
return rows.map((row) => {
const base = {
createdAt: new Date(row.created_at),
description: row.description,
id: row.id,
relevance: Number(row.relevance),
title: row.title,
type: row.type as SearchResultType,
updatedAt: new Date(row.updated_at),
};
switch (row.type) {
case 'page': {
return {
...base,
type: 'page' as const,
};
}
case 'pageContent': {
return {
...base,
type: 'pageContent' as const,
};
}
case 'agent': {
// Parse tags JSONB if string
let tags: string[] = [];
if (row.tags) {
if (typeof row.tags === 'string') {
try {
tags = JSON.parse(row.tags);
} catch {
tags = [];
}
} else {
tags = row.tags;
}
}
return {
...base,
avatar: row.avatar,
backgroundColor: row.background_color,
slug: row.slug,
tags,
type: 'agent' as const,
};
}
case 'topic': {
return {
...base,
agentId: row.agent_id,
favorite: row.favorite,
sessionId: row.session_id,
type: 'topic' as const,
};
}
case 'file': {
return {
...base,
fileType: row.file_type,
knowledgeBaseId: row.knowledge_base_id,
name: row.name,
size: Number(row.size),
type: 'file' as const,
url: row.url,
};
}
case 'message': {
return {
...base,
agentId: row.agent_id,
content: row.description || '',
model: row.slug,
role: row.name || 'user',
topicId: row.session_id,
type: 'message' as const,
};
}
default: {
throw new Error(`Unknown search result type: ${row.type}`);
}
}
});
}
}
+5
View File
@@ -50,6 +50,11 @@ export interface LobeDocument {
*/
pages?: LobeDocumentPage[];
/**
* Parent Folder ID
*/
parentId?: string | null;
/**
* Full path of the original file
*/
+1
View File
@@ -68,6 +68,7 @@ export const HotkeyEnum = {
OpenChatSettings: 'openChatSettings',
OpenHotkeyHelper: 'openHotkeyHelper',
RegenerateMessage: 'regenerateMessage',
SaveDocument: 'saveDocument',
SaveTopic: 'saveTopic',
Search: 'search',
ShowApp: 'showApp',
-2
View File
@@ -1,2 +0,0 @@
export type { DMTagProps } from './index';
export { default } from './index';
-79
View File
@@ -1,79 +0,0 @@
import { Tag } from '@lobehub/ui';
import { Lock } from 'lucide-react';
import { memo } from 'react';
import { useTranslation } from 'react-i18next';
import { DEFAULT_INBOX_AVATAR } from '@/const/meta';
import { useChatStore } from '@/store/chat';
import { useChatGroupStore } from '@/store/chatGroup';
import { useSessionStore } from '@/store/session';
import { sessionMetaSelectors } from '@/store/session/selectors';
import { useUserStore } from '@/store/user';
import { userProfileSelectors } from '@/store/user/selectors';
export interface DMTagProps {
/**
* ID of the message sender - can be agent ID or "user"
*/
senderId?: string;
/**
* ID of the message target - can be agent ID or "user"
*/
targetId?: string;
}
const DMTag = memo<DMTagProps>(({ senderId, targetId }) => {
const { t } = useTranslation('chat');
const toggleThread = useChatGroupStore((s) => s.toggleThread);
const togglePortal = useChatStore((s) => s.togglePortal);
const currentUserAvatar = useUserStore(userProfileSelectors.userAvatar);
const targetInfo = useSessionStore((s) => {
if (!targetId) return null;
if (targetId === 'user') {
return {
avatar: currentUserAvatar || DEFAULT_INBOX_AVATAR,
backgroundColor: undefined,
name: t('you'),
};
}
const agentMeta = sessionMetaSelectors.getAgentMetaByAgentId(targetId)(s);
return {
avatar: agentMeta.avatar || DEFAULT_INBOX_AVATAR,
backgroundColor: agentMeta.backgroundColor,
name: ` ${agentMeta.title || t('untitledAgent')} `,
};
});
// Don't show tag if we don't have target info
if (!targetInfo) return null;
// Check if message involves user (either sent by user or sent to user)
// const involvesUser = senderId === 'user' || targetId === 'user';
// Handler for opening thread panel
const handleOpenThread = () => {
// Open thread with the non-user participant
const agentId = senderId === 'user' ? targetId : senderId;
if (agentId && agentId !== 'user') {
toggleThread(agentId);
togglePortal(true);
}
};
return (
<Tag
color="default"
icon={<Lock size={12} />}
onClick={handleOpenThread}
size="small"
style={{ cursor: 'pointer' }}
>
{t('messages.dm.sentTo', { name: targetInfo.name })}
</Tag>
);
});
export default DMTag;
+49
View File
@@ -0,0 +1,49 @@
'use client';
import { Icon, Tag } from '@lobehub/ui';
import dayjs from 'dayjs';
import relativeTime from 'dayjs/plugin/relativeTime';
import { CloudIcon, Loader2Icon } from 'lucide-react';
import { CSSProperties, memo } from 'react';
import { useTranslation } from 'react-i18next';
dayjs.extend(relativeTime);
interface AutoSaveHintProps {
lastUpdatedTime?: Date | null;
saveStatus: 'idle' | 'saving' | 'saved';
style?: CSSProperties;
}
/**
* AutoSaveHint - Unified save status indicator for editors
*
* Displays real-time save status for document/config changes
*/
const AutoSaveHint = memo<AutoSaveHintProps>(({ style, saveStatus, lastUpdatedTime }) => {
const { t } = useTranslation('editor');
const isSaving = saveStatus === 'saving';
if (isSaving)
return (
<Tag icon={<Icon icon={Loader2Icon} spin />} style={style}>
{t('autoSave.saving')}
</Tag>
);
if (saveStatus === 'saved' && lastUpdatedTime)
return (
<Tag icon={<Icon icon={CloudIcon} />} style={style}>
{t('autoSave.saved')} {dayjs(lastUpdatedTime).fromNow()}
</Tag>
);
return (
<Tag icon={<Icon icon={CloudIcon} />} style={style}>
{t('autoSave.latest')}
</Tag>
);
});
export default AutoSaveHint;
+1 -1
View File
@@ -2,7 +2,7 @@ import { memo } from 'react';
import { Center } from 'react-layout-kit';
import FileIcon from '@/components/FileIcon';
import RepoIcon from '@/components/RepoIcon';
import RepoIcon from '@/components/LibIcon';
import { KnowledgeType } from '@/types/knowledgeBase';
interface KnowledgeIconProps {
@@ -5,10 +5,6 @@ import { memo } from 'react';
import { useTranslation } from 'react-i18next';
import { Center, Flexbox } from 'react-layout-kit';
import { SupervisorTodoItem } from '@/store/chat/slices/message/supervisor';
import { useSessionStore } from '@/store/session';
import { sessionSelectors } from '@/store/session/selectors';
const useStyles = createStyles(({ css, token }) => ({
collapse: css`
padding-block: 0;
@@ -29,29 +25,28 @@ const useStyles = createStyles(({ css, token }) => ({
`,
}));
export interface TodoData {
timestamp: number;
todos: SupervisorTodoItem[];
type: 'supervisor_todo';
export interface TodoItem {
assignee?: string;
content: string;
finished?: boolean;
}
export interface TodoListProps {
data: TodoData;
/**
* Optional function to resolve assignee ID to display name
*/
resolveAssigneeName?: (assignee: string) => string | undefined;
/**
* List of todo items
*/
todos: TodoItem[];
}
const TodoList = memo<TodoListProps>(({ data }) => {
const TodoList = memo<TodoListProps>(({ todos, resolveAssigneeName }) => {
const { t } = useTranslation('chat');
const theme = useTheme();
const { styles } = useStyles();
const { todos } = data;
const agents = useSessionStore(sessionSelectors.currentGroupAgents);
const resolveAssigneeName = (assignee?: string) => {
if (!assignee) return undefined;
const agent = agents?.find((a) => a.id === assignee);
return agent?.title || assignee;
};
const completedCount = todos.filter((todo) => todo.finished).length;
const totalCount = todos.length;
@@ -116,7 +111,7 @@ const TodoList = memo<TodoListProps>(({ data }) => {
>
{todo.content}
</span>
{resolveAssigneeName(todo.assignee) && (
{todo.assignee && (
<span
style={{
color: theme.colorTextTertiary,
@@ -124,7 +119,7 @@ const TodoList = memo<TodoListProps>(({ data }) => {
marginLeft: 6,
}}
>
@{resolveAssigneeName(todo.assignee)}
@{resolveAssigneeName?.(todo.assignee) ?? todo.assignee}
</span>
)}
</Flexbox>
+60 -3
View File
@@ -1,6 +1,7 @@
import { z } from 'zod';
import { ChunkModel } from '@/database/models/chunk';
import { DocumentModel } from '@/database/models/document';
import { FileModel } from '@/database/models/file';
import { MessageModel } from '@/database/models/message';
import { authedProcedure, router } from '@/libs/trpc/lambda';
@@ -13,6 +14,7 @@ const documentProcedure = authedProcedure.use(serverDatabase).use(async (opts) =
return opts.next({
ctx: {
chunkModel: new ChunkModel(ctx.serverDB, ctx.userId),
documentModel: new DocumentModel(ctx.serverDB, ctx.userId),
documentService: new DocumentService(ctx.serverDB, ctx.userId),
fileModel: new FileModel(ctx.serverDB, ctx.userId),
messageModel: new MessageModel(ctx.serverDB, ctx.userId),
@@ -29,15 +31,27 @@ export const documentRouter = router({
fileType: z.string().optional(),
knowledgeBaseId: z.string().optional(),
metadata: z.record(z.any()).optional(),
parentId: z.string().optional(),
slug: z.string().optional(),
title: z.string(),
}),
)
.mutation(async ({ ctx, input }) => {
// Resolve parentId if it's a slug
let resolvedParentId = input.parentId;
if (input.parentId) {
const docBySlug = await ctx.documentModel.findBySlug(input.parentId);
if (docBySlug) {
resolvedParentId = docBySlug.id;
}
}
// Parse editorData from JSON string to object
const editorData = JSON.parse(input.editorData);
return ctx.documentService.createDocument({
...input,
editorData,
parentId: resolvedParentId,
});
}),
@@ -47,12 +61,43 @@ export const documentRouter = router({
return ctx.documentService.deleteDocument(input.id);
}),
deleteDocuments: documentProcedure
.input(z.object({ ids: z.array(z.string()) }))
.mutation(async ({ ctx, input }) => {
return ctx.documentService.deleteDocuments(input.ids);
}),
getDocumentById: documentProcedure
.input(z.object({ id: z.string() }))
.query(async ({ ctx, input }) => {
return ctx.documentService.getDocumentById(input.id);
}),
getFolderBreadcrumb: documentProcedure
.input(z.object({ slug: z.string() }))
.query(async ({ ctx, input }) => {
const chain = [];
let currentFolder = await ctx.documentModel.findBySlug(input.slug);
// Build chain from current folder to root
while (currentFolder) {
chain.unshift({
id: currentFolder.id,
name: currentFolder.title || currentFolder.filename || 'Untitled',
slug: currentFolder.slug || currentFolder.id,
});
// Find parent folder
if (currentFolder.parentId) {
currentFolder = await ctx.documentModel.findById(currentFolder.parentId);
} else {
break;
}
}
return chain;
}),
parseFileContent: documentProcedure
.input(
z.object({
@@ -66,9 +111,20 @@ export const documentRouter = router({
return lobeDocument;
}),
queryDocuments: documentProcedure.query(async ({ ctx }) => {
return ctx.documentService.queryDocuments();
}),
queryDocuments: documentProcedure
.input(
z
.object({
current: z.number().optional(),
fileTypes: z.array(z.string()).optional(),
pageSize: z.number().optional(),
sourceTypes: z.array(z.string()).optional(),
})
.optional(),
)
.query(async ({ ctx, input }) => {
return ctx.documentService.queryDocuments(input);
}),
updateDocument: documentProcedure
.input(
@@ -77,6 +133,7 @@ export const documentRouter = router({
editorData: z.string().optional(),
id: z.string(),
metadata: z.record(z.any()).optional(),
parentId: z.string().nullable().optional(),
rawData: z.string().optional(),
title: z.string().optional(),
}),
+136 -5
View File
@@ -36,10 +36,24 @@ export const fileRouter = router({
}),
createFile: fileProcedure
.input(UploadFileSchema.omit({ url: true }).extend({ url: z.string() }))
.input(
UploadFileSchema.omit({ url: true }).extend({
parentId: z.string().optional(),
url: z.string(),
}),
)
.mutation(async ({ ctx, input }) => {
const { isExist } = await ctx.fileModel.checkHash(input.hash!);
// Resolve parentId if it's a slug
let resolvedParentId = input.parentId;
if (input.parentId) {
const docBySlug = await ctx.documentModel.findBySlug(input.parentId);
if (docBySlug) {
resolvedParentId = docBySlug.id;
}
}
const { id } = await ctx.fileModel.create(
{
fileHash: input.hash,
@@ -47,6 +61,7 @@ export const fileRouter = router({
knowledgeBaseId: input.knowledgeBaseId,
metadata: input.metadata,
name: input.name,
parentId: resolvedParentId,
size: input.size,
url: input.url,
},
@@ -169,10 +184,28 @@ export const fileRouter = router({
}),
getKnowledgeItems: fileProcedure.input(QueryFileListSchema).query(async ({ ctx, input }) => {
const knowledgeItems = await ctx.knowledgeRepo.query(input);
// Request one more item than limit to check if there are more items
const limit = input.limit ?? 50;
const knowledgeItems = await ctx.knowledgeRepo.query({
...input,
limit: limit + 1,
});
// Check if there are more items
const hasMore = knowledgeItems.length > limit;
// Take only the requested number of items
const itemsToProcess = hasMore ? knowledgeItems.slice(0, limit) : knowledgeItems;
// Filter out folders from Documents category when in Inbox (no knowledgeBaseId)
const filteredItems = !input.knowledgeBaseId
? itemsToProcess.filter(
(item) => !(item.sourceType === 'document' && item.fileType === 'custom/folder'),
)
: itemsToProcess;
// Process files (add chunk info and async task status)
const fileItems = knowledgeItems.filter((item) => item.sourceType === 'file');
const fileItems = filteredItems.filter((item) => item.sourceType === 'file');
const fileIds = fileItems.map((item) => item.id);
const chunks = await ctx.chunkModel.countByFileIds(fileIds);
@@ -189,7 +222,7 @@ export const fileRouter = router({
// Combine all items with their metadata
const resultItems = [] as any[];
for (const item of knowledgeItems) {
for (const item of filteredItems) {
if (item.sourceType === 'file') {
const chunkTask = item.chunkTaskId
? chunkTasks.find((task) => task.id === item.chunkTaskId)
@@ -224,9 +257,83 @@ export const fileRouter = router({
}
}
return resultItems;
return {
hasMore,
items: resultItems,
};
}),
recentFiles: fileProcedure
.input(z.object({ limit: z.number().optional() }).optional())
.query(async ({ ctx, input }) => {
const limit = input?.limit ?? 12;
// Query recent items and filter for files only (exclude documents/pages)
const allItems = await ctx.knowledgeRepo.queryRecent(limit * 3); // Query more to ensure we have enough files after filtering
const fileItems = allItems
.filter((item) => item.sourceType === 'file' && item.fileType !== 'custom/document')
.slice(0, limit);
if (fileItems.length === 0) return [];
// Get file IDs for batch processing
const fileIds = fileItems.map((item) => item.id);
const chunksArray = await ctx.chunkModel.countByFileIds(fileIds);
const chunks: Record<string, number> = {};
for (const item of chunksArray) {
if (item.id) chunks[item.id] = item.count;
}
const chunkTaskIds = fileItems.map((item) => item.chunkTaskId).filter(Boolean) as string[];
const embeddingTaskIds = fileItems
.map((item) => item.embeddingTaskId)
.filter(Boolean) as string[];
const [chunkTasks, embeddingTasks] = await Promise.all([
chunkTaskIds.length > 0
? ctx.asyncTaskModel.findByIds(chunkTaskIds, AsyncTaskType.Chunking)
: Promise.resolve([]),
embeddingTaskIds.length > 0
? ctx.asyncTaskModel.findByIds(embeddingTaskIds, AsyncTaskType.Embedding)
: Promise.resolve([]),
]);
// Build result with task status
const resultFiles: FileListItem[] = [];
for (const item of fileItems) {
const chunkTask = item.chunkTaskId
? chunkTasks.find((task) => task.id === item.chunkTaskId)
: null;
const embeddingTask = item.embeddingTaskId
? embeddingTasks.find((task) => task.id === item.embeddingTaskId)
: null;
resultFiles.push({
...item,
chunkCount: chunks[item.id] ?? 0,
chunkingError: chunkTask?.error ?? null,
chunkingStatus: chunkTask?.status as AsyncTaskStatus,
embeddingError: embeddingTask?.error ?? null,
embeddingStatus: embeddingTask?.status as AsyncTaskStatus,
finishEmbedding: embeddingTask?.status === AsyncTaskStatus.Success,
sourceType: 'file' as const,
url: await ctx.fileService.getFullFileUrl(item.url!),
} as FileListItem);
}
return resultFiles;
}),
recentPages: fileProcedure
.input(z.object({ limit: z.number().optional() }).optional())
.query(async ({ ctx, input }) => {
const limit = input?.limit ?? 12;
// Query recent items and filter for pages (documents) only, exclude folders
const allItems = await ctx.knowledgeRepo.queryRecent(limit * 3); // Query more to ensure we have enough pages after filtering
return allItems
.filter((item) => item.sourceType === 'document' && item.fileType !== 'custom/folder')
.slice(0, limit);
}),
removeAllFiles: fileProcedure.mutation(async ({ ctx }) => {
return ctx.fileModel.clear();
}),
@@ -272,6 +379,30 @@ export const fileRouter = router({
// remove from S3
await ctx.fileService.deleteFiles(needToRemoveFileList.map((file) => file.url!));
}),
updateFile: fileProcedure
.input(
z.object({
id: z.string(),
parentId: z.string().nullable().optional(),
}),
)
.mutation(async ({ ctx, input }) => {
const { id, parentId } = input;
// Resolve parentId if it's a slug (otherwise use as-is)
let resolvedParentId: string | null | undefined = parentId;
if (parentId) {
const docBySlug = await ctx.documentModel.findBySlug(parentId);
if (docBySlug) {
resolvedParentId = docBySlug.id;
}
}
await ctx.fileModel.update(id, { parentId: resolvedParentId });
return { success: true };
}),
});
export type FileRouter = typeof fileRouter;
+177
View File
@@ -0,0 +1,177 @@
import { z } from 'zod';
import { SearchRepo } from '@/database/repositories/search';
import { authedProcedure, router } from '@/libs/trpc/lambda';
import { serverDatabase } from '@/libs/trpc/lambda/middleware';
import { DiscoverService } from '@/server/services/discover';
/**
* Calculate relevance score for marketplace items
* 1 = exact match, 2 = prefix match, 3 = contains match
*/
function calculateMarketplaceRelevance(query: string, title: string): number {
const lowerQuery = query.toLowerCase().trim();
const lowerTitle = title.toLowerCase();
if (lowerTitle === lowerQuery) return 1;
if (lowerTitle.startsWith(lowerQuery)) return 2;
if (lowerTitle.includes(lowerQuery)) return 3;
return 4;
}
const searchProcedure = authedProcedure.use(serverDatabase).use(async (opts) => {
const { ctx } = opts;
return opts.next({
ctx: {
discoverService: new DiscoverService({ accessToken: ctx.marketAccessToken }),
searchRepo: new SearchRepo(ctx.serverDB, ctx.userId),
},
});
});
/**
* The unified search router for all entities in the database.
*
* Can specify the type of entity to search for.
*/
export const searchRouter = router({
query: searchProcedure
.input(
z.object({
agentId: z.string().optional(),
limitPerType: z.number().optional(),
locale: z.string().optional(),
offset: z.number().optional(),
query: z.string(),
type: z
.enum(['agent', 'topic', 'file', 'message', 'mcp', 'plugin', 'assistant'])
.optional(),
}),
)
.query(async ({ input, ctx }) => {
const { query, type, limitPerType = 5, locale } = input;
// Early return for empty query
if (!query || query.trim() === '') return [];
// Build search promises based on type filter
const searchPromises: Promise<any>[] = [];
// Database searches (agent, topic, file, message)
if (!type || ['agent', 'topic', 'file', 'message'].includes(type)) {
searchPromises.push(ctx.searchRepo.search(input));
}
// Marketplace searches (mcp, plugin)
if (!type || type === 'mcp') {
searchPromises.push(
ctx.discoverService
.getMcpList({
locale,
pageSize: limitPerType,
q: query,
})
.then((response) =>
response.items.slice(0, limitPerType).map((item: any) => ({
author:
typeof item.author === 'string' ? item.author : item.author?.name || 'Unknown',
avatar: item.avatar || item.icon || null,
category: item.category || null,
connectionType: item.connectionType || null,
createdAt: new Date(item.createdAt || Date.now()),
description: item.description || null,
id: item.identifier,
identifier: item.identifier,
installCount: item.installCount || null,
isFeatured: item.isFeatured || null,
isValidated: item.isValidated || null,
relevance: calculateMarketplaceRelevance(
query,
(item.name || item.title || item.identifier) as string,
),
tags: item.tags || null,
title: (item.name || item.title || item.identifier) as string,
type: 'mcp' as const,
updatedAt: new Date(item.updatedAt || Date.now()),
})),
)
.catch(() => []),
);
}
if (!type || type === 'plugin') {
searchPromises.push(
ctx.discoverService
.getPluginList({
locale,
pageSize: limitPerType,
q: query,
})
.then((response) =>
response.items.slice(0, limitPerType).map((item: any) => ({
author:
typeof item.author === 'string' ? item.author : item.author?.name || 'Unknown',
avatar: item.avatar || null,
category: item.category || null,
createdAt: new Date(item.createdAt || Date.now()),
description: item.description || null,
id: item.identifier,
identifier: item.identifier,
relevance: calculateMarketplaceRelevance(
query,
(item.title || item.identifier) as string,
),
tags: item.tags || null,
title: (item.title || item.identifier) as string,
type: 'plugin' as const,
updatedAt: new Date(item.updatedAt || Date.now()),
})),
)
.catch(() => []),
);
}
if (!type || type === 'assistant') {
searchPromises.push(
ctx.discoverService
.getAssistantList({
locale,
pageSize: limitPerType,
q: query,
})
.then((response) =>
response.items.slice(0, limitPerType).map((item: any) => ({
author:
typeof item.author === 'string' ? item.author : item.author?.name || 'Unknown',
avatar: item.avatar || null,
createdAt: new Date(item.createdAt || Date.now()),
description: item.description || null,
homepage: item.homepage || null,
id: item.identifier,
identifier: item.identifier,
relevance: calculateMarketplaceRelevance(
query,
(item.title || item.identifier) as string,
),
tags: item.tags || null,
title: (item.title || item.identifier) as string,
type: 'assistant' as const,
updatedAt: new Date(item.updatedAt || Date.now()),
})),
)
.catch(() => []),
);
}
// Execute searches in parallel and merge results
const results = await Promise.all(searchPromises);
const mergedResults = results.flat();
// Sort by relevance and limit total results
return mergedResults.sort((a, b) => {
if (a.relevance !== b.relevance) return a.relevance - b.relevance;
return new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime();
});
}),
});
+101 -8
View File
@@ -1,7 +1,8 @@
import { LobeChatDatabase } from '@lobechat/database';
import { DocumentItem } from '@lobechat/database/schemas';
import { DocumentItem, documents, files } from '@lobechat/database/schemas';
import { loadFile } from '@lobechat/file-loaders';
import debug from 'debug';
import { and, eq } from 'drizzle-orm';
import { DocumentModel } from '@/database/models/document';
import { FileModel } from '@/database/models/file';
@@ -35,7 +36,9 @@ export class DocumentService {
fileType?: string;
knowledgeBaseId?: string;
metadata?: Record<string, any>;
parentId?: string;
rawData?: string;
slug?: string;
title: string;
}): Promise<DocumentItem> {
const {
@@ -45,20 +48,48 @@ export class DocumentService {
fileType = 'custom/document',
metadata,
knowledgeBaseId,
parentId,
slug,
} = params;
// Calculate character and line counts
const totalCharCount = content?.length || 0;
const totalLineCount = content?.split('\n').length || 0;
let fileId: string | null = null;
// If creating in a knowledge base, create a corresponding file record
// BUT skip for folders - folders should only exist in the documents table
if (knowledgeBaseId && fileType !== 'custom/folder') {
const file = await this.fileModel.create(
{
fileType,
knowledgeBaseId,
metadata,
name: title,
parentId,
size: totalCharCount,
url: `internal://document/placeholder`, // Placeholder URL
},
false, // Do not insert to global files
);
fileId = file.id;
}
// Store knowledgeBaseId in metadata for folders (which don't have fileId)
const finalMetadata =
knowledgeBaseId && fileType === 'custom/folder' ? { ...metadata, knowledgeBaseId } : metadata;
const document = await this.documentModel.create({
content,
editorData,
fileId: knowledgeBaseId ? null : undefined,
fileId,
fileType,
filename: title,
metadata,
metadata: finalMetadata,
pages: undefined,
parentId,
slug,
source: 'document',
sourceType: 'api',
title,
@@ -70,10 +101,15 @@ export class DocumentService {
}
/**
* Query all documents
* Query documents with pagination
*/
async queryDocuments() {
return this.documentModel.query();
async queryDocuments(params?: {
current?: number;
fileTypes?: string[];
pageSize?: number;
sourceTypes?: string[];
}) {
return this.documentModel.query(params);
}
/**
@@ -84,12 +120,50 @@ export class DocumentService {
}
/**
* Delete document
* Delete document (recursively deletes children if it's a folder)
*/
async deleteDocument(id: string) {
const document = await this.documentModel.findById(id);
if (!document) return;
// If it's a folder, recursively delete all children first
if (document.fileType === 'custom/folder') {
const children = await this.db.query.documents.findMany({
where: eq(documents.parentId, id),
});
// Recursively delete all children
for (const child of children) {
await this.deleteDocument(child.id);
}
// Also delete all files in this folder
const childFiles = await this.db.query.files.findMany({
where: and(eq(files.parentId, id), eq(files.userId, this.userId)),
});
for (const file of childFiles) {
await this.fileModel.delete(file.id);
}
}
// Delete the associated file record if it exists
if (document.fileId) {
await this.fileModel.delete(document.fileId);
}
// Finally delete the document itself
return this.documentModel.delete(id);
}
/**
* Delete multiple documents in batch
*/
async deleteDocuments(ids: string[]) {
// Delete each document (which handles recursive deletion for folders)
await Promise.all(ids.map((id) => this.deleteDocument(id)));
}
/**
* Update document
*/
@@ -99,6 +173,7 @@ export class DocumentService {
content?: string;
editorData?: Record<string, any>;
metadata?: Record<string, any>;
parentId?: string | null;
title?: string;
},
) {
@@ -123,7 +198,24 @@ export class DocumentService {
updates.metadata = params.metadata;
}
return this.documentModel.update(id, updates);
if (params.parentId !== undefined) {
updates.parentId = params.parentId;
}
const result = await this.documentModel.update(id, updates);
// If title was updated and this document has an associated file, update the file name too
if (params.title !== undefined || params.parentId !== undefined) {
const document = await this.documentModel.findById(id);
if (document?.fileId) {
const fileUpdates: any = {};
if (params.title !== undefined) fileUpdates.name = params.title;
if (params.parentId !== undefined) fileUpdates.parentId = params.parentId;
await this.fileModel.update(document.fileId, fileUpdates);
}
}
return result;
}
/**
@@ -151,6 +243,7 @@ export class DocumentService {
fileType: file.fileType,
metadata: fileDocument.metadata,
pages: fileDocument.pages,
parentId: file.parentId,
source: file.url,
sourceType: 'file',
title: fileDocument.metadata?.title,
+2 -1
View File
@@ -1,5 +1,5 @@
import { SearchParams, SearchQuery } from '@lobechat/types';
import { CrawlImplType, Crawler } from '@lobechat/web-crawler';
import { CrawlImplType } from '@lobechat/web-crawler';
import pMap from 'p-map';
import { toolsEnv } from '@/envs/tools';
@@ -30,6 +30,7 @@ export class SearchService {
}
async crawlPages(input: { impls?: CrawlImplType[]; urls: string[] }) {
const { Crawler } = await import('@lobechat/web-crawler');
const crawler = new Crawler({ impls: this.crawlerImpls });
const results = await pMap(