♻️ refactor: refactor database schema (#10860)

* update data schema

* update data schema

docs/development/database-schema.dbml

@@ -8,7 +8,7 @@ table agents {
   avatar text
   background_color text
   market_identifier text
-  plugins jsonb [default: `[]`]
+  plugins jsonb
   client_id text
   user_id text [not null]
   chat_config jsonb
@@ -19,6 +19,7 @@ table agents {
   system_role text
   tts jsonb
   virtual boolean [default: false]
+  pinned boolean
   opening_message text
   opening_questions text[] [default: `[]`]
   accessed_at "timestamp with time zone" [not null, default: `now()`]
@@ -88,6 +89,7 @@ table ai_models {
 
   indexes {
     (id, provider_id, user_id) [pk]
+    user_id [name: 'ai_models_user_id_idx']
   }
 }
 
@@ -111,6 +113,7 @@ table ai_providers {
 
   indexes {
     (id, user_id) [pk]
+    user_id [name: 'ai_providers_user_id_idx']
   }
 }
 
@@ -421,6 +424,9 @@ table message_groups {
   parent_message_id text
   title varchar(255)
   description text
+  type text
+  content text
+  editor_data jsonb
   client_id varchar(255)
   accessed_at "timestamp with time zone" [not null, default: `now()`]
   created_at "timestamp with time zone" [not null, default: `now()`]
@@ -429,6 +435,7 @@ table message_groups {
   indexes {
     (client_id, user_id) [name: 'message_groups_client_id_user_id_unique', unique]
     topic_id [name: 'message_groups_topic_id_idx']
+    type [name: 'message_groups_type_idx']
   }
 }
 
@@ -447,6 +454,7 @@ table message_plugins {
 
   indexes {
     (client_id, user_id) [name: 'message_plugins_client_id_user_id_unique', unique]
+    tool_call_id [name: 'message_plugins_tool_call_id_idx']
   }
 }
 
@@ -507,6 +515,7 @@ table messages {
   role varchar(255) [not null]
   content text
   editor_data jsonb
+  summary text
   reasoning jsonb
   search jsonb
   metadata jsonb
@@ -997,6 +1006,9 @@ table threads {
   source_message_id text
   parent_thread_id text
   client_id text
+  agent_id text
+  group_id text
+  metadata jsonb
   user_id text [not null]
   last_active_at "timestamp with time zone" [default: `now()`]
   accessed_at "timestamp with time zone" [not null, default: `now()`]
@@ -1006,6 +1018,8 @@ table threads {
   indexes {
     (client_id, user_id) [name: 'threads_client_id_user_id_unique', unique]
     topic_id [name: 'threads_topic_id_idx']
+    agent_id [name: 'threads_agent_id_idx']
+    group_id [name: 'threads_group_id_idx']
   }
 }
 
@@ -1075,6 +1089,7 @@ table user_settings {
   system_agent jsonb
   default_agent jsonb
   market jsonb
+  memory jsonb
   tool jsonb
   image jsonb
 }
@@ -1089,7 +1104,9 @@ table users {
   first_name text
   last_name text
   full_name text
+  interests "varchar(64)[]"
   is_onboarded boolean [default: false]
+  onboarding jsonb
   clerk_created_at "timestamp with time zone"
   email_verified boolean [not null, default: false]
   email_verified_at "timestamp with time zone"
@@ -1150,7 +1167,6 @@ table user_memories_contexts {
   associated_objects jsonb
   associated_subjects jsonb
   title text
-  title_vector vector(1024)
   description text
   description_vector vector(1024)
   type varchar(255)
@@ -1163,7 +1179,6 @@ table user_memories_contexts {
   updated_at "timestamp with time zone" [not null, default: `now()`]
 
   indexes {
-    title_vector [name: 'user_memories_contexts_title_vector_index']
     description_vector [name: 'user_memories_contexts_description_vector_index']
     type [name: 'user_memories_contexts_type_index']
     user_id [name: 'user_memories_contexts_user_id_index']

package.json

@@ -60,7 +60,7 @@
     "e2e:install": "playwright install",
     "e2e:ui": "playwright test --ui",
     "i18n": "npm run workflow:i18n && lobe-i18n && prettier -c --write \"locales/**\"",
-    "lint": "npm run lint:ts && npm run lint:style && npm run typecheck && npm run lint:circular",
+    "lint": "npm run lint:ts && npm run lint:style && npm run type-check && npm run lint:circular",
     "lint:circular": "npm run lint:circular:main && npm run lint:circular:packages",
     "lint:circular:main": "dpdm src/**/*.ts  --no-warning --no-tree --exit-code circular:1 --no-progress -T true --skip-dynamic-imports circular",
     "lint:circular:packages": "dpdm packages/**/src/**/*.ts  --no-warning --no-tree --exit-code circular:1 --no-progress -T true --skip-dynamic-imports circular",
@@ -85,7 +85,7 @@
     "test:e2e": "pnpm --filter @lobechat/e2e-tests test",
     "test:e2e:smoke": "pnpm --filter @lobechat/e2e-tests test:smoke",
     "test:update": "vitest -u",
-    "typecheck": "tsgo --noEmit",
+    "type-check": "tsgo --noEmit",
     "webhook:ngrok": "ngrok http http://localhost:3011",
     "workflow:cdn": "tsx ./scripts/cdnWorkflow/index.ts",
     "workflow:changelog": "tsx ./scripts/changelogWorkflow/index.ts",

packages/database/migrations/0063_add_columns_for_several_tables.sql

@@ -0,0 +1,30 @@
+DROP INDEX IF EXISTS "user_memories_contexts_title_vector_index";--> statement-breakpoint
+ALTER TABLE "agents" ALTER COLUMN "plugins" DROP DEFAULT;--> statement-breakpoint
+ALTER TABLE "agents" ADD COLUMN IF NOT EXISTS "pinned" boolean;--> statement-breakpoint
+ALTER TABLE "message_groups" ADD COLUMN IF NOT EXISTS "type" text;--> statement-breakpoint
+ALTER TABLE "message_groups" ADD COLUMN IF NOT EXISTS "content" text;--> statement-breakpoint
+ALTER TABLE "message_groups" ADD COLUMN IF NOT EXISTS "editor_data" jsonb;--> statement-breakpoint
+ALTER TABLE "messages" ADD COLUMN IF NOT EXISTS "summary" text;--> statement-breakpoint
+ALTER TABLE "threads" ADD COLUMN IF NOT EXISTS "agent_id" text;--> statement-breakpoint
+ALTER TABLE "threads" ADD COLUMN IF NOT EXISTS "group_id" text;--> statement-breakpoint
+ALTER TABLE "threads" ADD COLUMN IF NOT EXISTS "metadata" jsonb;--> statement-breakpoint
+ALTER TABLE "user_settings" ADD COLUMN IF NOT EXISTS "memory" jsonb;--> statement-breakpoint
+ALTER TABLE "users" ADD COLUMN IF NOT EXISTS "interests" varchar(64)[];--> statement-breakpoint
+ALTER TABLE "users" ADD COLUMN IF NOT EXISTS "onboarding" jsonb;--> statement-breakpoint
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'threads_agent_id_agents_id_fk') THEN
+    ALTER TABLE "threads" ADD CONSTRAINT "threads_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE cascade ON UPDATE no action;
+  END IF;
+END $$;--> statement-breakpoint
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'threads_group_id_chat_groups_id_fk') THEN
+    ALTER TABLE "threads" ADD CONSTRAINT "threads_group_id_chat_groups_id_fk" FOREIGN KEY ("group_id") REFERENCES "public"."chat_groups"("id") ON DELETE cascade ON UPDATE no action;
+  END IF;
+END $$;--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "ai_models_user_id_idx" ON "ai_models" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "ai_providers_user_id_idx" ON "ai_providers" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "message_groups_type_idx" ON "message_groups" USING btree ("type");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "message_plugins_tool_call_id_idx" ON "message_plugins" USING btree ("tool_call_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "threads_agent_id_idx" ON "threads" USING btree ("agent_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "threads_group_id_idx" ON "threads" USING btree ("group_id");--> statement-breakpoint
+ALTER TABLE "user_memories_contexts" DROP COLUMN IF EXISTS "title_vector";

packages/database/migrations/meta/_journal.json

@@ -441,6 +441,13 @@
       "when": 1765728439737,
       "tag": "0062_add_more_index",
       "breakpoints": true
+    },
+    {
+      "idx": 63,
+      "version": "7",
+      "when": 1766157362540,
+      "tag": "0063_add_columns_for_several_tables",
+      "breakpoints": true
     }
   ],
   "version": "6"

packages/database/src/core/migrations.json

@@ -994,5 +994,34 @@
     "bps": true,
     "folderMillis": 1765728439737,
     "hash": "ba6a6beff2ad39419ec4aa7887539c9db0472e13f9242cb8780ff96eec450268"
+  },
+  {
+    "sql": [
+      "DROP INDEX IF EXISTS \"user_memories_contexts_title_vector_index\";",
+      "\nALTER TABLE \"agents\" ALTER COLUMN \"plugins\" DROP DEFAULT;",
+      "\nALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"pinned\" boolean;",
+      "\nALTER TABLE \"message_groups\" ADD COLUMN IF NOT EXISTS \"type\" text;",
+      "\nALTER TABLE \"message_groups\" ADD COLUMN IF NOT EXISTS \"content\" text;",
+      "\nALTER TABLE \"message_groups\" ADD COLUMN IF NOT EXISTS \"editor_data\" jsonb;",
+      "\nALTER TABLE \"messages\" ADD COLUMN IF NOT EXISTS \"summary\" text;",
+      "\nALTER TABLE \"threads\" ADD COLUMN IF NOT EXISTS \"agent_id\" text;",
+      "\nALTER TABLE \"threads\" ADD COLUMN IF NOT EXISTS \"group_id\" text;",
+      "\nALTER TABLE \"threads\" ADD COLUMN IF NOT EXISTS \"metadata\" jsonb;",
+      "\nALTER TABLE \"user_settings\" ADD COLUMN IF NOT EXISTS \"memory\" jsonb;",
+      "\nALTER TABLE \"users\" ADD COLUMN IF NOT EXISTS \"interests\" varchar(64)[];",
+      "\nALTER TABLE \"users\" ADD COLUMN IF NOT EXISTS \"onboarding\" jsonb;",
+      "\nDO $$ BEGIN\n  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'threads_agent_id_agents_id_fk') THEN\n    ALTER TABLE \"threads\" ADD CONSTRAINT \"threads_agent_id_agents_id_fk\" FOREIGN KEY (\"agent_id\") REFERENCES \"public\".\"agents\"(\"id\") ON DELETE cascade ON UPDATE no action;\n  END IF;\nEND $$;",
+      "\nDO $$ BEGIN\n  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'threads_group_id_chat_groups_id_fk') THEN\n    ALTER TABLE \"threads\" ADD CONSTRAINT \"threads_group_id_chat_groups_id_fk\" FOREIGN KEY (\"group_id\") REFERENCES \"public\".\"chat_groups\"(\"id\") ON DELETE cascade ON UPDATE no action;\n  END IF;\nEND $$;",
+      "\nCREATE INDEX IF NOT EXISTS \"ai_models_user_id_idx\" ON \"ai_models\" USING btree (\"user_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"ai_providers_user_id_idx\" ON \"ai_providers\" USING btree (\"user_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"message_groups_type_idx\" ON \"message_groups\" USING btree (\"type\");",
+      "\nCREATE INDEX IF NOT EXISTS \"message_plugins_tool_call_id_idx\" ON \"message_plugins\" USING btree (\"tool_call_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"threads_agent_id_idx\" ON \"threads\" USING btree (\"agent_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"threads_group_id_idx\" ON \"threads\" USING btree (\"group_id\");",
+      "\nALTER TABLE \"user_memories_contexts\" DROP COLUMN IF EXISTS \"title_vector\";\n"
+    ],
+    "bps": true,
+    "folderMillis": 1766157362540,
+    "hash": "7a8ee107778222390e676951173baa81bfa09dd47216a8467575fca54915172c"
   }
 ]

packages/database/src/schemas/_helpers.ts

@@ -10,7 +10,11 @@ export const updatedAt = () =>
     .notNull()
     .defaultNow()
     .$onUpdate(() => new Date());
-export const accessedAt = () => timestamptz('accessed_at').notNull().defaultNow();
+export const accessedAt = () =>
+  timestamptz('accessed_at')
+    .notNull()
+    .defaultNow()
+    .$onUpdate(() => new Date());
 
 // columns.helpers.ts
 export const timestamps = {

packages/database/src/schemas/agent.ts

@@ -37,7 +37,7 @@ export const agents = pgTable(
     backgroundColor: text('background_color'),
     marketIdentifier: text('market_identifier'),
 
-    plugins: jsonb('plugins').$type<string[]>().default([]),
+    plugins: jsonb('plugins').$type<string[]>(),
 
     clientId: text('client_id'),
 
@@ -55,6 +55,7 @@ export const agents = pgTable(
     tts: jsonb('tts').$type<LobeAgentTTSConfig>(),
 
     virtual: boolean('virtual').default(false),
+    pinned: boolean('pinned'),
 
     openingMessage: text('opening_message'),
     openingQuestions: text('opening_questions').array().default([]),

packages/database/src/schemas/aiInfra.ts

@@ -1,6 +1,15 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
 import type { AiProviderConfig, AiProviderSettings } from '@lobechat/types';
-import { boolean, integer, jsonb, pgTable, primaryKey, text, varchar } from 'drizzle-orm/pg-core';
+import {
+  boolean,
+  index,
+  integer,
+  jsonb,
+  pgTable,
+  primaryKey,
+  text,
+  varchar,
+} from 'drizzle-orm/pg-core';
 import { AiModelSettings } from 'model-bank';
 
 import { timestamps } from './_helpers';
@@ -36,7 +45,10 @@ export const aiProviders = pgTable(
 
     ...timestamps,
   },
-  (table) => [primaryKey({ columns: [table.id, table.userId] })],
+  (table) => [
+    primaryKey({ columns: [table.id, table.userId] }),
+    index('ai_providers_user_id_idx').on(table.userId),
+  ],
 );
 
 export type NewAiProviderItem = Omit<typeof aiProviders.$inferInsert, 'userId'>;
@@ -68,7 +80,10 @@ export const aiModels = pgTable(
 
     ...timestamps,
   },
-  (table) => [primaryKey({ columns: [table.id, table.providerId, table.userId] })],
+  (table) => [
+    primaryKey({ columns: [table.id, table.providerId, table.userId] }),
+    index('ai_models_user_id_idx').on(table.userId),
+  ],
 );
 
 export type NewAiModelItem = Omit<typeof aiModels.$inferInsert, 'userId'>;

packages/database/src/schemas/message.ts

@@ -58,6 +58,11 @@ export const messageGroups = pgTable(
     title: varchar255('title'),
     description: text('description'),
 
+    // Compression fields
+    type: text('type', { enum: ['parallel', 'compression'] }),
+    content: text('content'), // compression summary (plain text)
+    editorData: jsonb('editor_data'), // rich text editor data (future extension)
+
     clientId: varchar255('client_id'),
 
     ...timestamps,
@@ -65,6 +70,7 @@ export const messageGroups = pgTable(
   (t) => [
     uniqueIndex('message_groups_client_id_user_id_unique').on(t.clientId, t.userId),
     index('message_groups_topic_id_idx').on(t.topicId),
+    index('message_groups_type_idx').on(t.type),
   ],
 );
 
@@ -84,6 +90,7 @@ export const messages = pgTable(
     role: varchar255('role').notNull(),
     content: text('content'),
     editorData: jsonb('editor_data'),
+    summary: text('summary'),
     reasoning: jsonb('reasoning').$type<ModelReasoning>(),
     search: jsonb('search').$type<GroundingSearch>(),
     metadata: jsonb('metadata'),
@@ -164,12 +171,10 @@ export const messagePlugins = pgTable(
       .references(() => users.id, { onDelete: 'cascade' })
       .notNull(),
   },
-  (t) => ({
-    clientIdUnique: uniqueIndex('message_plugins_client_id_user_id_unique').on(
-      t.clientId,
-      t.userId,
-    ),
-  }),
+  (t) => [
+    uniqueIndex('message_plugins_client_id_user_id_unique').on(t.clientId, t.userId),
+    index('message_plugins_tool_call_id_idx').on(t.toolCallId),
+  ],
 );
 
 export const messageTTS = pgTable(

packages/database/src/schemas/topic.ts

@@ -1,5 +1,5 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
-import type { ChatTopicMetadata } from '@lobechat/types';
+import type { ChatTopicMetadata, ThreadMetadata } from '@lobechat/types';
 import { sql } from 'drizzle-orm';
 import { boolean, index, jsonb, pgTable, primaryKey, text, uniqueIndex } from 'drizzle-orm/pg-core';
 import { createInsertSchema } from 'drizzle-zod';
@@ -63,7 +63,16 @@ export const threads = pgTable(
     editor_data: jsonb('editor_data'),
     type: text('type', { enum: ['continuation', 'standalone', 'isolation'] }).notNull(),
     status: text('status', {
-      enum: ['active', 'processing', 'pending', 'inReview', 'todo', 'cancel'],
+      enum: [
+        'active',
+        'processing',
+        'pending',
+        'inReview',
+        'todo',
+        'cancel',
+        'completed',
+        'failed',
+      ],
     }),
 
     topicId: text('topic_id')
@@ -74,6 +83,10 @@ export const threads = pgTable(
     parentThreadId: text('parent_thread_id').references(() => threads.id, { onDelete: 'set null' }),
     clientId: text('client_id'),
 
+    agentId: text('agent_id').references(() => agents.id, { onDelete: 'cascade' }),
+    groupId: text('group_id').references(() => chatGroups.id, { onDelete: 'cascade' }),
+    metadata: jsonb('metadata').$type<ThreadMetadata | undefined>(),
+
     userId: text('user_id')
       .references(() => users.id, { onDelete: 'cascade' })
       .notNull(),
@@ -84,6 +97,8 @@ export const threads = pgTable(
   (t) => [
     uniqueIndex('threads_client_id_user_id_unique').on(t.clientId, t.userId),
     index('threads_topic_id_idx').on(t.topicId),
+    index('threads_agent_id_idx').on(t.agentId),
+    index('threads_group_id_idx').on(t.groupId),
   ],
 );
 

packages/database/src/schemas/user.ts

@@ -1,9 +1,9 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
 import { DEFAULT_PREFERENCE } from '@lobechat/const';
-import type { CustomPluginParams } from '@lobechat/types';
+import type { CustomPluginParams, UserOnboarding } from '@lobechat/types';
 import { LobeChatPluginManifest } from '@lobehub/chat-plugin-sdk';
 import { sql } from 'drizzle-orm';
-import { boolean, index, jsonb, pgTable, primaryKey, text } from 'drizzle-orm/pg-core';
+import { boolean, index, jsonb, pgTable, primaryKey, text, varchar } from 'drizzle-orm/pg-core';
 
 import { timestamps, timestamptz, varchar255 } from './_helpers';
 
@@ -20,8 +20,10 @@ export const users = pgTable(
     firstName: text('first_name'),
     lastName: text('last_name'),
     fullName: text('full_name'),
+    interests: varchar('interests', { length: 64 }).array(),
 
     isOnboarded: boolean('is_onboarded').default(false),
+    onboarding: jsonb('onboarding').$type<UserOnboarding>(),
     // Time user was created in Clerk
     clerkCreatedAt: timestamptz('clerk_created_at'),
 
@@ -77,6 +79,7 @@ export const userSettings = pgTable('user_settings', {
   systemAgent: jsonb('system_agent'),
   defaultAgent: jsonb('default_agent'),
   market: jsonb('market'),
+  memory: jsonb('memory'),
   tool: jsonb('tool'),
   image: jsonb('image'),
 });

packages/database/src/schemas/userMemories.ts

@@ -55,16 +55,21 @@ export const userMemoriesContexts = pgTable(
       .primaryKey(),
 
     userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
-    userMemoryIds: jsonb('user_memory_ids'),
+    userMemoryIds: jsonb('user_memory_ids').$type<string[]>(),
 
     metadata: jsonb('metadata').$type<Record<string, unknown>>(),
     tags: text('tags').array(),
 
-    associatedObjects: jsonb('associated_objects'),
-    associatedSubjects: jsonb('associated_subjects'),
+    associatedObjects:
+      jsonb('associated_objects').$type<
+        { extra?: Record<string, unknown>; name?: string; type?: string }[]
+      >(),
+    associatedSubjects:
+      jsonb('associated_subjects').$type<
+        { extra?: Record<string, unknown>; name?: string; type?: string }[]
+      >(),
 
     title: text('title'),
-    titleVector: vector('title_vector', { dimensions: 1024 }),
     description: text('description'),
     descriptionVector: vector('description_vector', { dimensions: 1024 }),
 
@@ -79,10 +84,6 @@ export const userMemoriesContexts = pgTable(
     ...timestamps,
   },
   (table) => [
-    index('user_memories_contexts_title_vector_index').using(
-      'hnsw',
-      table.titleVector.op('vector_cosine_ops'),
-    ),
     index('user_memories_contexts_description_vector_index').using(
       'hnsw',
       table.descriptionVector.op('vector_cosine_ops'),

packages/types/src/topic/thread.ts

@@ -25,6 +25,30 @@ export interface ThreadItem {
   userId: string;
 }
 
+/**
+ * Metadata for Thread, used for agent task execution
+ */
+export interface ThreadMetadata {
+  /** Task completion time */
+  completedAt?: string;
+  /** Execution duration in milliseconds */
+  duration?: number;
+  /** Error message when task failed */
+  error?: string;
+  /** Operation ID for tracking */
+  operationId?: string;
+  /** Task start time, used to calculate duration */
+  startedAt?: string;
+  /** Total cost in dollars */
+  totalCost?: number;
+  /** Total messages created during execution */
+  totalMessages?: number;
+  /** Total tokens consumed */
+  totalTokens?: number;
+  /** Total tool calls made */
+  totalToolCalls?: number;
+}
+
 export interface CreateThreadParams {
   parentThreadId?: string;
   sourceMessageId?: string;

packages/types/src/user/index.ts

@@ -1,2 +1,3 @@
+export * from './onboarding';
 export * from './preference';
 export * from './settings';

packages/types/src/user/onboarding.ts

@@ -0,0 +1,16 @@
+import { z } from 'zod';
+
+export interface UserOnboarding {
+  /** Current step number (1-based), for resuming onboarding */
+  currentStep?: number;
+  /** Timestamp when onboarding was completed (ISO 8601) */
+  finishedAt?: string;
+  /** Onboarding flow version for future upgrades */
+  version: number;
+}
+
+export const UserOnboardingSchema = z.object({
+  currentStep: z.number().optional(),
+  finishedAt: z.string().optional(),
+  version: z.number(),
+});