From 2bb1506ea25a60685413a5b5969f1847e4539d04 Mon Sep 17 00:00:00 2001 From: YuTengjing Date: Wed, 16 Jul 2025 16:36:43 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix:=20chat=20model=20list=20sho?= =?UTF-8?q?uld=20not=20show=20image=20model=20(#8448)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cursor/rules/project-introduce.mdc | 57 +- .cursor/rules/testing-guide.mdc | 881 ------------------ .cursor/rules/testing-guide/db-model-test.mdc | 453 +++++++++ .../rules/testing-guide/electron-ipc-test.mdc | 80 ++ .cursor/rules/testing-guide/testing-guide.mdc | 401 ++++++++ package.json | 2 +- src/config/aiModels/openai.ts | 33 +- src/libs/model-runtime/BaseAI.ts | 1 + src/libs/model-runtime/hunyuan/index.ts | 10 +- .../novita/__snapshots__/index.test.ts.snap | 18 + .../openai/__snapshots__/index.test.ts.snap | 28 + src/libs/model-runtime/openai/index.test.ts | 339 +------ src/libs/model-runtime/openai/index.ts | 127 --- .../__snapshots__/index.test.ts.snap | 3 + .../ppio/__snapshots__/index.test.ts.snap | 2 + src/libs/model-runtime/utils/modelParse.ts | 1 + .../openaiCompatibleFactory/index.test.ts | 376 +++++++- .../utils/openaiCompatibleFactory/index.ts | 192 +++- .../model-runtime/utils/openaiHelpers.test.ts | 151 +++ src/libs/model-runtime/utils/openaiHelpers.ts | 27 +- src/libs/model-runtime/xai/index.ts | 5 +- src/store/aiInfra/slices/aiModel/action.ts | 2 +- src/store/aiInfra/slices/aiProvider/action.ts | 7 +- src/types/aiModel.ts | 1 + src/types/llm.ts | 4 +- 25 files changed, 1717 insertions(+), 1484 deletions(-) delete mode 100644 .cursor/rules/testing-guide.mdc create mode 100644 .cursor/rules/testing-guide/db-model-test.mdc create mode 100644 .cursor/rules/testing-guide/electron-ipc-test.mdc create mode 100644 .cursor/rules/testing-guide/testing-guide.mdc diff --git a/.cursor/rules/project-introduce.mdc b/.cursor/rules/project-introduce.mdc index b0e57ccc27..448c42428c 100644 --- a/.cursor/rules/project-introduce.mdc +++ b/.cursor/rules/project-introduce.mdc @@ -55,59 +55,4 @@ pnpm install # !: don't any build script to check weather code can work after modify ``` -check [testing guide](./testing-guide.mdc) to learn test scripts. - -## Project Description - -You are developing an open-source, modern-design AI chat framework: lobe chat. - -Emoji logo: 🤯 - -## Project Technologies Stack - -read [package.json](mdc:package.json) to know all npm packages you can use. read [folder-structure.mdx](mdc:docs/development/basic/folder-structure.mdx) to learn project structure. - -The project uses the following technologies: - -- pnpm as package manager -- Next.js 15 for frontend and backend, using app router instead of pages router -- react 19, using hooks, functional components, react server components -- TypeScript programming language -- antd, @lobehub/ui for component framework -- antd-style for css-in-js framework -- react-layout-kit for flex layout -- react-i18next for i18n -- lucide-react, @ant-design/icons for icons -- @lobehub/icons for AI provider/model logo icon -- @formkit/auto-animate for react list animation -- zustand for global state management -- nuqs for type-safe search params state manager -- SWR for react data fetch -- aHooks for react hooks library -- dayjs for date and time library -- lodash-es for utility library -- fast-deep-equal for deep comparison of JavaScript objects -- zod for data validation -- TRPC for type safe backend -- PGLite for client DB and PostgreSQL for backend DB -- Drizzle ORM -- Vitest for testing, testing-library for react component test -- Prettier for code formatting -- ESLint for code linting -- Cursor AI for code editing and AI coding assistance - -Note: All tools and libraries used are the latest versions. The application only needs to be compatible with the latest browsers; - -## Often used npm scripts - -```bash -# type check -bun type-check - -# install dependencies -pnpm install - -# !: don't any build script to check weather code can work after modify -``` - -check [testing guide](./testing-guide.mdc) to learn test scripts. +check [testing guide](./testing-guide/testing-guide.mdc) to learn test scripts. diff --git a/.cursor/rules/testing-guide.mdc b/.cursor/rules/testing-guide.mdc deleted file mode 100644 index 7045cb949c..0000000000 --- a/.cursor/rules/testing-guide.mdc +++ /dev/null @@ -1,881 +0,0 @@ ---- -description: -globs: *.test.ts,*.test.tsx -alwaysApply: false ---- ---- -type: agent-requested -title: 测试指南 - LobeChat Testing Guide -description: LobeChat 项目的 Vitest 测试环境配置、运行方式、修复原则指南 ---- - -# 测试指南 - LobeChat Testing Guide - -## 🧪 测试环境概览 - -LobeChat 项目使用 Vitest 测试库，配置了两种不同的测试环境： - -### 客户端测试环境 (DOM Environment) - -- **配置文件**: [vitest.config.ts](mdc:vitest.config.ts) -- **环境**: Happy DOM (浏览器环境模拟) -- **数据库**: PGLite (浏览器环境的 PostgreSQL) -- **用途**: 测试前端组件、客户端逻辑、React 组件等 -- **设置文件**: [tests/setup.ts](mdc:tests/setup.ts) - -### 服务端测试环境 (Node Environment) - -- **配置文件**: [vitest.config.server.ts](mdc:vitest.config.server.ts) -- **环境**: Node.js -- **数据库**: 真实的 PostgreSQL 数据库 -- **并发限制**: 单线程运行 (`singleFork: true`) -- **用途**: 测试数据库模型、服务端逻辑、API 端点等 -- **设置文件**: [tests/setup-db.ts](mdc:tests/setup-db.ts) - -## 🚀 测试运行命令 - -### package.json 脚本说明 - -查看 [package.json](mdc:package.json) 中的测试相关脚本： - -```json -{ - "test": "npm run test-app && npm run test-server", - "test-app": "vitest run --config vitest.config.ts", - "test-app:coverage": "vitest run --config vitest.config.ts --coverage", - "test-server": "vitest run --config vitest.config.server.ts", - "test-server:coverage": "vitest run --config vitest.config.server.ts --coverage" -} -``` - -### 推荐的测试运行方式 - -#### ✅ 正确的命令格式 - -```bash -# 运行所有客户端测试 -npx vitest run --config vitest.config.ts - -# 运行所有服务端测试 -npx vitest run --config vitest.config.server.ts - -# 运行特定测试文件 (支持模糊匹配) -npx vitest run --config vitest.config.ts basic -npx vitest run --config vitest.config.ts user.test.ts - -# 运行特定文件的特定行号 -npx vitest run --config vitest.config.ts src/utils/helper.test.ts:25 -npx vitest run --config vitest.config.ts basic/foo.test.ts:10,basic/foo.test.ts:25 - -# 过滤特定测试用例名称 -npx vitest -t "test case name" --config vitest.config.ts - -# 组合使用文件和测试名称过滤 -npx vitest run --config vitest.config.ts filename.test.ts -t "specific test" -``` - -#### ❌ 避免的命令格式 - -```bash -# ❌ 不要使用 pnpm test xxx (这不是有效的 vitest 命令) -pnpm test some-file - -# ❌ 不要使用裸 vitest (会进入 watch 模式) -vitest test-file.test.ts - -# ❌ 不要混淆测试环境 -npx vitest run --config vitest.config.server.ts client-component.test.ts -``` - -### 关键运行参数说明 - -- **`vitest run`**: 运行一次测试然后退出 (避免 watch 模式) -- **`vitest`**: 默认进入 watch 模式，持续监听文件变化 -- **`--config`**: 指定配置文件，选择正确的测试环境 -- **`-t`**: 过滤测试用例名称，支持正则表达式 -- **`--coverage`**: 生成测试覆盖率报告 - -## 🔧 测试修复原则 - -### 核心原则 ⚠️ - -1. **充分阅读测试代码**: 在修复测试之前，必须完整理解测试的意图和实现 -2. **测试优先修复**: 如果是测试本身写错了，修改测试而不是实现代码 -3. **专注单一问题**: 只修复指定的测试，不要添加额外测试或功能 -4. **不自作主张**: 不要因为发现其他问题就直接修改，先提出再讨论 - -### 测试修复流程 - -```mermaid -flowchart TD - subgraph "阶段一：分析与复现" - A[开始：收到测试失败报告] --> B[定位并运行失败的测试]; - B --> C{是否能在本地复现?}; - C -->|否| D[检查测试环境/配置/依赖]; - C -->|是| E[分析：阅读测试代码、错误日志、Git 历史]; - end - - subgraph "阶段二：诊断与调试" - E --> F[建立假设：问题出在测试、代码还是环境?]; - F --> G["调试：使用 console.log 或 debugger 深入检查"]; - G --> H{假设是否被证实?}; - H -->|否, 重新假设| F; - end - - subgraph "阶段三：修复与验证" - H -->|是| I{确定根本原因}; - I -->|测试逻辑错误| J[修复测试代码]; - I -->|实现代码 Bug| K[修复实现代码]; - I -->|环境/配置问题| L[修复配置或依赖]; - J --> M[验证修复：重新运行失败的测试]; - K --> M; - L --> M; - M --> N{测试是否通过?}; - N -->|否, 修复无效| F; - N -->|是| O[扩大验证：运行当前文件内所有测试]; - O --> P{是否全部通过?}; - P -->|否, 引入新问题| F; - end - - subgraph "阶段四：总结" - P -->|是| Q[完成：撰写修复总结]; - end - - D --> F; -``` - -### 修复完成后的总结 - -测试修复完成后，应该提供简要说明，包括： - -1. **错误原因分析**: 说明测试失败的根本原因 - - 测试逻辑错误 - - 实现代码bug - - 环境配置问题 - - 依赖变更导致的问题 - -2. **修复方法说明**: 简述采用的修复方式 - - 修改了哪些文件 - - 采用了什么解决方案 - - 为什么选择这种修复方式 - -**示例格式**: - -```markdown -## 测试修复总结 - -**错误原因**: 测试中的 mock 数据格式与实际 API 返回格式不匹配，导致断言失败。 - -**修复方法**: 更新了测试文件中的 mock 数据结构，使其与最新的 API 响应格式保持一致。具体修改了 `user.test.ts` 中的 `mockUserData` 对象结构。 -``` - -## 📂 测试文件组织 - -### 文件命名约定 - -- **客户端测试**: `*.test.ts`, `*.test.tsx` (任意位置) -- **服务端测试**: `src/database/models/**/*.test.ts`, `src/database/server/**/*.test.ts` (限定路径) - -### 测试文件组织风格 - -项目采用 **测试文件与源文件同目录** 的组织风格： - -- 测试文件放在对应源文件的同一目录下 -- 命名格式：`原文件名.test.ts` 或 `原文件名.test.tsx` - -例如： - -``` -src/components/Button/ -├── index.tsx # 源文件 -└── index.test.tsx # 测试文件 -``` - -## 🛠️ 测试调试技巧 - -### 运行失败测试的步骤 - -1. **确定测试类型**: 查看文件路径确定使用哪个配置 -2. **运行单个测试**: 使用 `-t` 参数隔离问题 -3. **检查错误日志**: 仔细阅读错误信息和堆栈跟踪 -4. **查看最近修改记录**: 检查相关文件的最近变更情况 -5. **添加调试日志**: 在测试中添加 `console.log` 了解执行流程 - -### Electron IPC 接口测试策略 🖥️ - -对于涉及 Electron IPC 接口的测试，由于提供真实的 Electron 环境比较复杂，采用 **Mock 返回值** 的方式进行测试。 - -#### 基本 Mock 设置 - -```typescript -import { vi } from "vitest"; -import { electronIpcClient } from "@/server/modules/ElectronIPCClient"; - -// Mock Electron IPC 客户端 -vi.mock("@/server/modules/ElectronIPCClient", () => ({ - electronIpcClient: { - getFilePathById: vi.fn(), - deleteFiles: vi.fn(), - // 根据需要添加其他 IPC 方法 - }, -})); -``` - -#### 在测试中设置 Mock 行为 - -```typescript -beforeEach(() => { - // 重置所有 Mock - vi.resetAllMocks(); - - // 设置默认的 Mock 返回值 - vi.mocked(electronIpcClient.getFilePathById).mockResolvedValue( - "/path/to/file.txt" - ); - vi.mocked(electronIpcClient.deleteFiles).mockResolvedValue({ - success: true, - }); -}); -``` - -#### 测试不同场景的示例 - -```typescript -it("应该处理文件删除成功的情况", async () => { - // 设置成功场景的 Mock - vi.mocked(electronIpcClient.deleteFiles).mockResolvedValue({ - success: true, - }); - - const result = await service.deleteFiles(["desktop://file1.txt"]); - - expect(electronIpcClient.deleteFiles).toHaveBeenCalledWith([ - "desktop://file1.txt", - ]); - expect(result.success).toBe(true); -}); - -it("应该处理文件删除失败的情况", async () => { - // 设置失败场景的 Mock - vi.mocked(electronIpcClient.deleteFiles).mockRejectedValue( - new Error("删除失败") - ); - - const result = await service.deleteFiles(["desktop://file1.txt"]); - - expect(result.success).toBe(false); - expect(result.errors).toBeDefined(); -}); -``` - -#### Mock 策略的优势 - -1. **环境简化**: 避免了复杂的 Electron 环境搭建 -2. **测试可控**: 可以精确控制 IPC 调用的返回值和行为 -3. **场景覆盖**: 容易测试各种成功/失败场景 -4. **执行速度**: Mock 调用比真实 IPC 调用更快 - -#### 注意事项 - -- **Mock 准确性**: 确保 Mock 的行为与真实 IPC 接口行为一致 -- **类型安全**: 使用 `vi.mocked()` 确保类型安全 -- **Mock 重置**: 在 `beforeEach` 中重置 Mock 状态，避免测试间干扰 -- **调用验证**: 不仅要验证返回值，还要验证 IPC 方法是否被正确调用 - -### 检查最近修改记录 🔍 - -为了更好地判断测试失败的根本原因，需要**系统性地检查相关文件的修改历史**。这是问题定位的关键步骤。 - -#### 第一步：确定需要检查的文件范围 - -1. **测试文件本身**: `path/to/component.test.ts` -2. **对应的实现文件**: `path/to/component.ts` 或 `path/to/component/index.ts` -3. **相关依赖文件**: 测试或实现中导入的其他模块 - -#### 第二步：检查当前工作目录状态 - -```bash -# 查看所有未提交的修改状态 -git status - -# 重点关注测试文件和实现文件是否有未提交的修改 -git status | grep -E "(test|spec)" -``` - -#### 第三步：检查未提交的修改内容 - -```bash -# 查看测试文件的未提交修改 (工作区 vs 暂存区) -git diff path/to/component.test.ts | cat - -# 查看对应实现文件的未提交修改 -git diff path/to/component.ts | cat - -# 查看已暂存但未提交的修改 -git diff --cached path/to/component.test.ts | cat -git diff --cached path/to/component.ts | cat -``` - -#### 第四步：检查提交历史和时间相关性 - -**首先查看提交时间，判断修改的时效性**： - -```bash -# 查看测试文件的最近提交历史，包含提交时间 -git log --pretty=format:"%h %ad %s" --date=relative -5 path/to/component.test.ts | cat - -# 查看实现文件的最近提交历史，包含提交时间 -git log --pretty=format:"%h %ad %s" --date=relative -5 path/to/component.ts | cat - -# 查看详细的提交时间（ISO格式，便于精确判断） -git log --pretty=format:"%h %ad %an %s" --date=iso -3 path/to/component.ts | cat -git log --pretty=format:"%h %ad %an %s" --date=iso -3 path/to/component.test.ts | cat -``` - -**判断提交的参考价值**： - -1. **最近提交（24小时内）**: 🔴 **高度相关** - 很可能是导致测试失败的直接原因 -2. **近期提交（1-7天内）**: 🟡 **中等相关** - 可能相关，需要仔细分析修改内容 -3. **较早提交（超过1周）**: ⚪ **低相关性** - 除非是重大重构，否则不太可能是直接原因 - -#### 第五步：基于时间相关性查看具体修改内容 - -**根据提交时间的远近，优先查看最近的修改**： - -```bash -# 如果有24小时内的提交，重点查看这些修改 -git show HEAD -- path/to/component.test.ts | cat -git show HEAD -- path/to/component.ts | cat - -# 查看次新的提交（如果最新提交时间较远） -git show HEAD~1 -- path/to/component.ts | cat -git show -- path/to/component.ts | cat - -# 对比最近两次提交的差异 -git diff HEAD~1 HEAD -- path/to/component.ts | cat -``` - -#### 第六步：分析修改与测试失败的关系 - -基于修改记录和时间相关性判断： - -1. **最近修改了实现代码**: - - ```bash - # 重点检查实现逻辑的变化 - git diff HEAD~1 path/to/component.ts | cat - ``` - - - 很可能是实现代码的变更导致测试失败 - - 检查实现逻辑是否正确 - - 确认测试是否需要相应更新 - -2. **最近修改了测试代码**: - - ```bash - # 重点检查测试逻辑的变化 - git diff HEAD~1 path/to/component.test.ts | cat - ``` - - - 可能是测试本身写错了 - - 检查测试逻辑和断言是否正确 - - 确认测试是否符合实现的预期行为 - -3. **两者都有最近修改**: - - ```bash - # 对比两个文件的修改时间 - git log --pretty=format:"%ad %f" --date=iso -1 path/to/component.ts | cat - git log --pretty=format:"%ad %f" --date=iso -1 path/to/component.test.ts | cat - ``` - - - 需要综合分析两者的修改 - - 确定哪个修改更可能导致问题 - - 优先检查时间更近的修改 - -4. **都没有最近修改**: - - 可能是依赖变更或环境问题 - - 检查 `package.json`、配置文件等的修改 - - 查看是否有全局性的代码重构 - -#### 修改记录检查示例 - -```bash -# 完整的检查流程示例 -echo "=== 检查文件修改状态 ===" -git status | grep component - -echo "=== 检查未提交修改 ===" -git diff src/components/Button/index.test.tsx | cat -git diff src/components/Button/index.tsx | cat - -echo "=== 检查提交历史和时间 ===" -git log --pretty=format:"%h %ad %s" --date=relative -3 src/components/Button/index.test.tsx | cat -git log --pretty=format:"%h %ad %s" --date=relative -3 src/components/Button/index.tsx | cat - -echo "=== 根据时间优先级查看修改内容 ===" -# 如果有24小时内的提交，重点查看 -git show HEAD -- src/components/Button/index.tsx | cat -``` - -## 🗃️ 数据库 Model 测试指南 - -### 测试环境选择 💡 - -数据库 Model 层通过环境变量控制数据库类型，在两种测试环境下有不同的数据库后端：客户端环境 (PGLite) 和 服务端环境 (PostgreSQL) - -### ⚠️ 双环境验证要求 - -**对于所有 Model 测试，必须在两个环境下都验证通过**： - -#### 完整验证流程 - -```bash -# 1. 先在客户端环境测试（快速验证） -npx vitest run --config vitest.config.ts src/database/models/__tests__/myModel.test.ts - -# 2. 再在服务端环境测试（兼容性验证） -npx vitest run --config vitest.config.server.ts src/database/models/__tests__/myModel.test.ts -``` - -### 创建新 Model 测试的最佳实践 📋 - -#### 1. 参考现有实现和测试模板 - -创建新 Model 测试前，**必须先参考现有的实现模式**： - -- **Model 实现参考**: -- **测试模板参考**: -- **复杂示例参考**: - -#### 2. 用户权限检查 - 安全第一 🔒 - -这是**最关键的安全要求**。所有涉及用户数据的操作都必须包含用户权限检查： - -**❌ 错误示例 - 存在安全漏洞**: - -```typescript -// 危险：缺少用户权限检查，任何用户都能操作任何数据 -update = async (id: string, data: Partial) => { - return this.db - .update(myTable) - .set(data) - .where(eq(myTable.id, id)) // ❌ 只检查 ID，没有检查 userId - .returning(); -}; -``` - -**✅ 正确示例 - 安全的实现**: - -```typescript -// 安全：必须同时匹配 ID 和 userId -update = async (id: string, data: Partial) => { - return this.db - .update(myTable) - .set(data) - .where( - and( - eq(myTable.id, id), - eq(myTable.userId, this.userId) // ✅ 用户权限检查 - ) - ) - .returning(); -}; -``` - -**必须进行用户权限检查的方法**： - -- `update()` - 更新操作 -- `delete()` - 删除操作 -- `findById()` - 查找特定记录 -- 任何涉及特定记录的查询或修改操作 - -#### 3. 测试文件结构和必测场景 - -**基本测试结构**: - -```typescript -// @vitest-environment node -describe("MyModel", () => { - describe("create", () => { - it("should create a new record"); - it("should handle edge cases"); - }); - - describe("queryAll", () => { - it("should return records for current user only"); - it("should handle empty results"); - }); - - describe("update", () => { - it("should update own records"); - it("should NOT update other users records"); // 🔒 安全测试 - }); - - describe("delete", () => { - it("should delete own records"); - it("should NOT delete other users records"); // 🔒 安全测试 - }); - - describe("user isolation", () => { - it("should enforce user data isolation"); // 🔒 核心安全测试 - }); -}); -``` - -**必须测试的安全场景** 🔒: - -```typescript -it("should not update records of other users", async () => { - // 创建其他用户的记录 - const [otherUserRecord] = await serverDB - .insert(myTable) - .values({ userId: "other-user", data: "original" }) - .returning(); - - // 尝试更新其他用户的记录 - const result = await myModel.update(otherUserRecord.id, { data: "hacked" }); - - // 应该返回 undefined 或空数组（因为权限检查失败） - expect(result).toBeUndefined(); - - // 验证原始数据未被修改 - const unchanged = await serverDB.query.myTable.findFirst({ - where: eq(myTable.id, otherUserRecord.id), - }); - expect(unchanged?.data).toBe("original"); // 数据应该保持不变 -}); -``` - -#### 4. Mock 外部依赖服务 - -如果 Model 依赖外部服务（如 FileService），需要正确 Mock： - -**设置 Mock**: - -```typescript -// 在文件顶部设置 Mock -const mockGetFullFileUrl = vi.fn(); -vi.mock("@/server/services/file", () => ({ - FileService: vi.fn().mockImplementation(() => ({ - getFullFileUrl: mockGetFullFileUrl, - })), -})); - -// 在 beforeEach 中重置和配置 Mock -beforeEach(async () => { - vi.clearAllMocks(); - mockGetFullFileUrl.mockImplementation( - (url: string) => `https://example.com/${url}` - ); -}); -``` - -**验证 Mock 调用**: - -```typescript -it("should process URLs through FileService", async () => { - // ... 测试逻辑 - - // 验证 Mock 被正确调用 - expect(mockGetFullFileUrl).toHaveBeenCalledWith("expected-url"); - expect(mockGetFullFileUrl).toHaveBeenCalledTimes(1); -}); -``` - -#### 5. 数据库状态管理 - -**正确的数据清理模式**: - -```typescript -const userId = "test-user"; -const otherUserId = "other-user"; - -beforeEach(async () => { - // 清理用户表（级联删除相关数据） - await serverDB.delete(users); - - // 创建测试用户 - await serverDB.insert(users).values([{ id: userId }, { id: otherUserId }]); -}); - -afterEach(async () => { - // 清理测试数据 - await serverDB.delete(users); -}); -``` - -#### 6. 测试数据类型和外键约束处理 ⚠️ - -**必须使用 Schema 导出的类型**: - -```typescript -// ✅ 正确：使用 schema 导出的类型 -import { NewGenerationBatch, NewGeneration } from '../../schemas'; - -const testBatch: NewGenerationBatch = { - userId, - generationTopicId: 'test-topic-id', - provider: 'test-provider', - model: 'test-model', - prompt: 'Test prompt for image generation', - width: 1024, - height: 1024, - config: { /* ... */ }, -}; - -const testGeneration: NewGeneration = { - id: 'test-gen-id', - generationBatchId: 'test-batch-id', - asyncTaskId: null, // 处理外键约束 - fileId: null, // 处理外键约束 - seed: 12345, - userId, -}; -``` - -```typescript -// ❌ 错误：没有类型声明或使用错误类型 -const testBatch = { // 缺少类型声明 - generationTopicId: 'test-topic-id', - // ... -}; - -const testGeneration = { // 缺少类型声明 - asyncTaskId: 'invalid-uuid', // 外键约束错误 - fileId: 'non-existent-file', // 外键约束错误 - // ... -}; -``` - -**外键约束处理策略**: - -1. **使用 null 值**: 对于可选的外键字段，使用 null 避免约束错误 -2. **创建关联记录**: 如果需要测试关联关系，先创建被引用的记录 -3. **理解约束关系**: 了解哪些字段有外键约束，避免引用不存在的记录 - -```typescript -// 外键约束处理示例 -beforeEach(async () => { - // 清理数据库 - await serverDB.delete(users); - - // 创建测试用户 - await serverDB.insert(users).values([{ id: userId }]); - - // 如果需要测试文件关联，创建文件记录 - if (needsFileAssociation) { - await serverDB.insert(files).values({ - id: 'test-file-id', - userId, - name: 'test.jpg', - url: 'test-url', - size: 1024, - fileType: 'image/jpeg', - }); - } -}); -``` - -**排序测试的可预测性**: - -```typescript -// ✅ 正确：使用明确的时间戳确保排序结果可预测 -it('should find batches by topic id in correct order', async () => { - const oldDate = new Date('2024-01-01T10:00:00Z'); - const newDate = new Date('2024-01-02T10:00:00Z'); - - const batch1 = { ...testBatch, prompt: 'First batch', userId, createdAt: oldDate }; - const batch2 = { ...testBatch, prompt: 'Second batch', userId, createdAt: newDate }; - - await serverDB.insert(generationBatches).values([batch1, batch2]); - - const results = await generationBatchModel.findByTopicId(testTopic.id); - - expect(results[0].prompt).toBe('Second batch'); // 最新优先 (desc order) - expect(results[1].prompt).toBe('First batch'); -}); -``` - -```typescript -// ❌ 错误：依赖数据库的默认时间戳，结果不可预测 -it('should find batches by topic id', async () => { - const batch1 = { ...testBatch, prompt: 'First batch', userId }; - const batch2 = { ...testBatch, prompt: 'Second batch', userId }; - - await serverDB.insert(generationBatches).values([batch1, batch2]); - - // 插入顺序和数据库时间戳可能不一致，导致测试不稳定 - const results = await generationBatchModel.findByTopicId(testTopic.id); - expect(results[0].prompt).toBe('Second batch'); // 可能失败 -}); -``` - - - -### 常见问题和解决方案 💡 - -#### 问题 1：权限检查缺失导致安全漏洞 - -**现象**: 测试失败，用户能修改其他用户的数据 -**解决**: 在 Model 的 `update` 和 `delete` 方法中添加 `and(eq(table.id, id), eq(table.userId, this.userId))` - -#### 问题 2：Mock 未生效或验证失败 - -**现象**: `undefined is not a spy` 错误 -**解决**: 检查 Mock 设置位置和方式，确保在测试文件顶部设置，在 `beforeEach` 中重置 - -#### 问题 3：测试数据污染 - -**现象**: 测试间相互影响，结果不稳定 -**解决**: 在 `beforeEach` 和 `afterEach` 中正确清理数据库状态 - -#### 问题 4：外部依赖导致测试失败 - -**现象**: 因为真实的外部服务调用导致测试不稳定 -**解决**: Mock 所有外部依赖，使测试更可控和快速 - -#### 问题 5：外键约束违反导致测试失败 - -**现象**: `insert or update on table "xxx" violates foreign key constraint` -**解决**: -- 将可选外键字段设为 `null` 而不是无效的字符串值 -- 或者先创建被引用的记录，再创建当前记录 - -```typescript -// ❌ 错误：无效的外键值 -const testData = { - asyncTaskId: 'invalid-uuid', // 表中不存在此记录 - fileId: 'non-existent-file', // 表中不存在此记录 -}; - -// ✅ 正确：使用 null 值 -const testData = { - asyncTaskId: null, // 避免外键约束 - fileId: null, // 避免外键约束 -}; - -// ✅ 或者：先创建被引用的记录 -beforeEach(async () => { - const [asyncTask] = await serverDB.insert(asyncTasks).values({ - id: 'valid-task-id', - status: 'pending', - type: 'generation', - }).returning(); - - const testData = { - asyncTaskId: asyncTask.id, // 使用有效的外键值 - }; -}); -``` - -#### 问题 6：排序测试结果不一致 - -**现象**: 相同的测试有时通过，有时失败，特别是涉及排序的测试 -**解决**: 使用明确的时间戳，不要依赖数据库的默认时间戳 - -```typescript -// ❌ 错误：依赖插入顺序和默认时间戳 -await serverDB.insert(table).values([data1, data2]); // 时间戳不可预测 - -// ✅ 正确：明确指定时间戳 -const oldDate = new Date('2024-01-01T10:00:00Z'); -const newDate = new Date('2024-01-02T10:00:00Z'); -await serverDB.insert(table).values([ - { ...data1, createdAt: oldDate }, - { ...data2, createdAt: newDate }, -]); -``` - -#### 问题 7：Mock 验证失败或调用次数不匹配 - -**现象**: `expect(mockFunction).toHaveBeenCalledWith(...)` 失败 -**解决**: -- 检查 Mock 函数的实际调用参数和期望参数是否完全匹配 -- 确认 Mock 在正确的时机被重置和配置 -- 使用 `toHaveBeenCalledTimes()` 验证调用次数 - -```typescript -// 在 beforeEach 中正确配置 Mock -beforeEach(() => { - vi.clearAllMocks(); // 重置所有 Mock - - mockGetFullFileUrl.mockImplementation((url: string) => `https://example.com/${url}`); - mockTransformGeneration.mockResolvedValue({ - id: 'test-id', - // ... 其他字段 - }); -}); - -// 测试中验证 Mock 调用 -it('should call FileService with correct parameters', async () => { - await model.someMethod(); - - // 验证调用参数 - expect(mockGetFullFileUrl).toHaveBeenCalledWith('expected-url'); - // 验证调用次数 - expect(mockGetFullFileUrl).toHaveBeenCalledTimes(1); -}); -``` - -### Model 测试检查清单 ✅ - -创建 Model 测试时，请确保以下各项都已完成： - -#### 🔧 基础配置 -- [ ] **双环境验证** - 在客户端环境 (vitest.config.ts) 和服务端环境 (vitest.config.server.ts) 下都测试通过 -- [ ] 参考了 `_template.ts` 和现有 Model 的实现模式 -- [ ] **使用正确的 Schema 类型** - 测试数据使用 `NewXxx` 类型声明，如 `NewGenerationBatch`、`NewGeneration` - -#### 🔒 安全测试 -- [ ] **所有涉及用户数据的操作都包含用户权限检查** -- [ ] 包含了用户权限隔离的安全测试 -- [ ] 测试了用户无法访问其他用户数据的场景 - -#### 🗃️ 数据处理 -- [ ] **正确处理外键约束** - 使用 `null` 值或先创建被引用记录 -- [ ] **排序测试使用明确时间戳** - 不依赖数据库默认时间，确保结果可预测 -- [ ] 在 `beforeEach` 和 `afterEach` 中正确管理数据库状态 -- [ ] 所有测试都能独立运行且互不干扰 - -#### 🎭 Mock 和外部依赖 -- [ ] 正确 Mock 了外部依赖服务 (如 FileService、GenerationModel) -- [ ] 在 `beforeEach` 中重置和配置 Mock -- [ ] 验证了 Mock 服务的调用参数和次数 -- [ ] 测试了外部服务错误场景的处理 - -#### 📋 测试覆盖 -- [ ] 测试覆盖了所有主要方法 (create, query, update, delete) -- [ ] 测试了边界条件和错误场景 -- [ ] 包含了空结果处理的测试 -- [ ] **确认两个环境下的测试结果一致** - -#### 🚨 常见问题检查 -- [ ] 没有外键约束违反错误 -- [ ] 排序测试结果稳定可预测 -- [ ] Mock 验证无失败 -- [ ] 无测试数据污染问题 - -### 安全警告 ⚠️ - -**数据库 Model 层是安全的第一道防线**。如果 Model 层缺少用户权限检查： - -1. **任何用户都能访问和修改其他用户的数据** -2. **即使上层有权限检查，也可能被绕过** -3. **可能导致严重的数据泄露和安全事故** - -因此，**每个涉及用户数据的 Model 方法都必须包含用户权限检查，且必须有对应的安全测试来验证这些检查的有效性**。 - -## 🎯 总结 - -修复测试时，记住以下关键点： - -- **使用正确的命令**: `npx vitest run --config [config-file]` -- **理解测试意图**: 先读懂测试再修复 -- **查看最近修改**: 检查相关文件的 git 修改记录，判断问题根源 -- **选择正确环境**: 客户端测试用 `vitest.config.ts`，服务端用 `vitest.config.server.ts` -- **专注单一问题**: 只修复当前的测试失败 -- **验证修复结果**: 确保修复后测试通过且无副作用 -- **提供修复总结**: 说明错误原因和修复方法 -- **Model 测试安全第一**: 必须包含用户权限检查和对应的安全测试 -- **Model 双环境验证**: 必须在 PGLite 和 PostgreSQL 两个环境下都验证通过 diff --git a/.cursor/rules/testing-guide/db-model-test.mdc b/.cursor/rules/testing-guide/db-model-test.mdc new file mode 100644 index 0000000000..293a8e8310 --- /dev/null +++ b/.cursor/rules/testing-guide/db-model-test.mdc @@ -0,0 +1,453 @@ +--- +globs: src/database/**/*.test.ts +alwaysApply: false +--- + +## 🗃️ 数据库 Model 测试指南 + +### 测试环境选择 💡 + +数据库 Model 层通过环境变量控制数据库类型，在两种测试环境下有不同的数据库后端：客户端环境 (PGLite) 和 服务端环境 (PostgreSQL) + +### ⚠️ 双环境验证要求 + +**对于所有 Model 测试，必须在两个环境下都验证通过**： + +#### 完整验证流程 + +```bash +# 1. 先在客户端环境测试（快速验证） +npx vitest run --config vitest.config.ts src/database/models/__tests__/myModel.test.ts + +# 2. 再在服务端环境测试（兼容性验证） +npx vitest run --config vitest.config.server.ts src/database/models/__tests__/myModel.test.ts +``` + +### 创建新 Model 测试的最佳实践 📋 + +#### 1. 参考现有实现和测试模板 + +创建新 Model 测试前，**必须先参考现有的实现模式**： + +- **Model 实现参考**: +- **测试模板参考**: +- **复杂示例参考**: + +#### 2. 用户权限检查 - 安全第一 🔒 + +这是**最关键的安全要求**。所有涉及用户数据的操作都必须包含用户权限检查： + +**❌ 错误示例 - 存在安全漏洞**: + +```typescript +// 危险：缺少用户权限检查，任何用户都能操作任何数据 +update = async (id: string, data: Partial) => { + return this.db + .update(myTable) + .set(data) + .where(eq(myTable.id, id)) // ❌ 只检查 ID，没有检查 userId + .returning(); +}; +``` + +**✅ 正确示例 - 安全的实现**: + +```typescript +// 安全：必须同时匹配 ID 和 userId +update = async (id: string, data: Partial) => { + return this.db + .update(myTable) + .set(data) + .where( + and( + eq(myTable.id, id), + eq(myTable.userId, this.userId), // ✅ 用户权限检查 + ), + ) + .returning(); +}; +``` + +**必须进行用户权限检查的方法**： + +- `update()` - 更新操作 +- `delete()` - 删除操作 +- `findById()` - 查找特定记录 +- 任何涉及特定记录的查询或修改操作 + +#### 3. 测试文件结构和必测场景 + +**基本测试结构**: + +```typescript +// @vitest-environment node +describe('MyModel', () => { + describe('create', () => { + it('should create a new record'); + it('should handle edge cases'); + }); + + describe('queryAll', () => { + it('should return records for current user only'); + it('should handle empty results'); + }); + + describe('update', () => { + it('should update own records'); + it('should NOT update other users records'); // 🔒 安全测试 + }); + + describe('delete', () => { + it('should delete own records'); + it('should NOT delete other users records'); // 🔒 安全测试 + }); + + describe('user isolation', () => { + it('should enforce user data isolation'); // 🔒 核心安全测试 + }); +}); +``` + +**必须测试的安全场景** 🔒: + +```typescript +it('should not update records of other users', async () => { + // 创建其他用户的记录 + const [otherUserRecord] = await serverDB + .insert(myTable) + .values({ userId: 'other-user', data: 'original' }) + .returning(); + + // 尝试更新其他用户的记录 + const result = await myModel.update(otherUserRecord.id, { data: 'hacked' }); + + // 应该返回 undefined 或空数组（因为权限检查失败） + expect(result).toBeUndefined(); + + // 验证原始数据未被修改 + const unchanged = await serverDB.query.myTable.findFirst({ + where: eq(myTable.id, otherUserRecord.id), + }); + expect(unchanged?.data).toBe('original'); // 数据应该保持不变 +}); +``` + +#### 4. Mock 外部依赖服务 + +如果 Model 依赖外部服务（如 FileService），需要正确 Mock： + +**设置 Mock**: + +```typescript +// 在文件顶部设置 Mock +const mockGetFullFileUrl = vi.fn(); +vi.mock('@/server/services/file', () => ({ + FileService: vi.fn().mockImplementation(() => ({ + getFullFileUrl: mockGetFullFileUrl, + })), +})); + +// 在 beforeEach 中重置和配置 Mock +beforeEach(async () => { + vi.clearAllMocks(); + mockGetFullFileUrl.mockImplementation((url: string) => `https://example.com/${url}`); +}); +``` + +**验证 Mock 调用**: + +```typescript +it('should process URLs through FileService', async () => { + // ... 测试逻辑 + + // 验证 Mock 被正确调用 + expect(mockGetFullFileUrl).toHaveBeenCalledWith('expected-url'); + expect(mockGetFullFileUrl).toHaveBeenCalledTimes(1); +}); +``` + +#### 5. 数据库状态管理 + +**正确的数据清理模式**: + +```typescript +const userId = 'test-user'; +const otherUserId = 'other-user'; + +beforeEach(async () => { + // 清理用户表（级联删除相关数据） + await serverDB.delete(users); + + // 创建测试用户 + await serverDB.insert(users).values([{ id: userId }, { id: otherUserId }]); +}); + +afterEach(async () => { + // 清理测试数据 + await serverDB.delete(users); +}); +``` + +#### 6. 测试数据类型和外键约束处理 ⚠️ + +**必须使用 Schema 导出的类型**: + +```typescript +// ✅ 正确：使用 schema 导出的类型 +import { NewGeneration, NewGenerationBatch } from '../../schemas'; + +const testBatch: NewGenerationBatch = { + userId, + generationTopicId: 'test-topic-id', + provider: 'test-provider', + model: 'test-model', + prompt: 'Test prompt for image generation', + width: 1024, + height: 1024, + config: { + /* ... */ + }, +}; + +const testGeneration: NewGeneration = { + id: 'test-gen-id', + generationBatchId: 'test-batch-id', + asyncTaskId: null, // 处理外键约束 + fileId: null, // 处理外键约束 + seed: 12345, + userId, +}; +``` + +```typescript +// ❌ 错误：没有类型声明或使用错误类型 +const testBatch = { + // 缺少类型声明 + generationTopicId: 'test-topic-id', + // ... +}; + +const testGeneration = { + // 缺少类型声明 + asyncTaskId: 'invalid-uuid', // 外键约束错误 + fileId: 'non-existent-file', // 外键约束错误 + // ... +}; +``` + +**外键约束处理策略**: + +1. **使用 null 值**: 对于可选的外键字段，使用 null 避免约束错误 +2. **创建关联记录**: 如果需要测试关联关系，先创建被引用的记录 +3. **理解约束关系**: 了解哪些字段有外键约束，避免引用不存在的记录 + +```typescript +// 外键约束处理示例 +beforeEach(async () => { + // 清理数据库 + await serverDB.delete(users); + + // 创建测试用户 + await serverDB.insert(users).values([{ id: userId }]); + + // 如果需要测试文件关联，创建文件记录 + if (needsFileAssociation) { + await serverDB.insert(files).values({ + id: 'test-file-id', + userId, + name: 'test.jpg', + url: 'test-url', + size: 1024, + fileType: 'image/jpeg', + }); + } +}); +``` + +**排序测试的可预测性**: + +```typescript +// ✅ 正确：使用明确的时间戳确保排序结果可预测 +it('should find batches by topic id in correct order', async () => { + const oldDate = new Date('2024-01-01T10:00:00Z'); + const newDate = new Date('2024-01-02T10:00:00Z'); + + const batch1 = { ...testBatch, prompt: 'First batch', userId, createdAt: oldDate }; + const batch2 = { ...testBatch, prompt: 'Second batch', userId, createdAt: newDate }; + + await serverDB.insert(generationBatches).values([batch1, batch2]); + + const results = await generationBatchModel.findByTopicId(testTopic.id); + + expect(results[0].prompt).toBe('Second batch'); // 最新优先 (desc order) + expect(results[1].prompt).toBe('First batch'); +}); +``` + +```typescript +// ❌ 错误：依赖数据库的默认时间戳，结果不可预测 +it('should find batches by topic id', async () => { + const batch1 = { ...testBatch, prompt: 'First batch', userId }; + const batch2 = { ...testBatch, prompt: 'Second batch', userId }; + + await serverDB.insert(generationBatches).values([batch1, batch2]); + + // 插入顺序和数据库时间戳可能不一致，导致测试不稳定 + const results = await generationBatchModel.findByTopicId(testTopic.id); + expect(results[0].prompt).toBe('Second batch'); // 可能失败 +}); +``` + +### 常见问题和解决方案 💡 + +#### 问题 1：权限检查缺失导致安全漏洞 + +**现象**: 测试失败，用户能修改其他用户的数据 **解决**: 在 Model 的 `update` 和 `delete` 方法中添加 `and(eq(table.id, id), eq(table.userId, this.userId))` + +#### 问题 2：Mock 未生效或验证失败 + +**现象**: `undefined is not a spy` 错误 **解决**: 检查 Mock 设置位置和方式，确保在测试文件顶部设置，在 `beforeEach` 中重置 + +#### 问题 3：测试数据污染 + +**现象**: 测试间相互影响，结果不稳定 **解决**: 在 `beforeEach` 和 `afterEach` 中正确清理数据库状态 + +#### 问题 4：外部依赖导致测试失败 + +**现象**: 因为真实的外部服务调用导致测试不稳定 **解决**: Mock 所有外部依赖，使测试更可控和快速 + +#### 问题 5：外键约束违反导致测试失败 + +**现象**: `insert or update on table "xxx" violates foreign key constraint` **解决**: + +- 将可选外键字段设为 `null` 而不是无效的字符串值 +- 或者先创建被引用的记录，再创建当前记录 + +```typescript +// ❌ 错误：无效的外键值 +const testData = { + asyncTaskId: 'invalid-uuid', // 表中不存在此记录 + fileId: 'non-existent-file', // 表中不存在此记录 +}; + +// ✅ 正确：使用 null 值 +const testData = { + asyncTaskId: null, // 避免外键约束 + fileId: null, // 避免外键约束 +}; + +// ✅ 或者：先创建被引用的记录 +beforeEach(async () => { + const [asyncTask] = await serverDB.insert(asyncTasks).values({ + id: 'valid-task-id', + status: 'pending', + type: 'generation', + }).returning(); + + const testData = { + asyncTaskId: asyncTask.id, // 使用有效的外键值 + }; +}); +``` + +#### 问题 6：排序测试结果不一致 + +**现象**: 相同的测试有时通过，有时失败，特别是涉及排序的测试 **解决**: 使用明确的时间戳，不要依赖数据库的默认时间戳 + +```typescript +// ❌ 错误：依赖插入顺序和默认时间戳 +await serverDB.insert(table).values([data1, data2]); // 时间戳不可预测 + +// ✅ 正确：明确指定时间戳 +const oldDate = new Date('2024-01-01T10:00:00Z'); +const newDate = new Date('2024-01-02T10:00:00Z'); +await serverDB.insert(table).values([ + { ...data1, createdAt: oldDate }, + { ...data2, createdAt: newDate }, +]); +``` + +#### 问题 7：Mock 验证失败或调用次数不匹配 + +**现象**: `expect(mockFunction).toHaveBeenCalledWith(...)` 失败 **解决**: + +- 检查 Mock 函数的实际调用参数和期望参数是否完全匹配 +- 确认 Mock 在正确的时机被重置和配置 +- 使用 `toHaveBeenCalledTimes()` 验证调用次数 + +```typescript +// 在 beforeEach 中正确配置 Mock +beforeEach(() => { + vi.clearAllMocks(); // 重置所有 Mock + + mockGetFullFileUrl.mockImplementation((url: string) => `https://example.com/${url}`); + mockTransformGeneration.mockResolvedValue({ + id: 'test-id', + // ... 其他字段 + }); +}); + +// 测试中验证 Mock 调用 +it('should call FileService with correct parameters', async () => { + await model.someMethod(); + + // 验证调用参数 + expect(mockGetFullFileUrl).toHaveBeenCalledWith('expected-url'); + // 验证调用次数 + expect(mockGetFullFileUrl).toHaveBeenCalledTimes(1); +}); +``` + +### Model 测试检查清单 ✅ + +创建 Model 测试时，请确保以下各项都已完成： + +#### 🔧 基础配置 + +- [ ] **双环境验证** - 在客户端环境 (vitest.config.ts) 和服务端环境 (vitest.config.server.ts) 下都测试通过 +- [ ] 参考了 `_template.ts` 和现有 Model 的实现模式 +- [ ] **使用正确的 Schema 类型** - 测试数据使用 `NewXxx` 类型声明，如 `NewGenerationBatch`、`NewGeneration` + +#### 🔒 安全测试 + +- [ ] **所有涉及用户数据的操作都包含用户权限检查** +- [ ] 包含了用户权限隔离的安全测试 +- [ ] 测试了用户无法访问其他用户数据的场景 + +#### 🗃️ 数据处理 + +- [ ] **正确处理外键约束** - 使用 `null` 值或先创建被引用记录 +- [ ] **排序测试使用明确时间戳** - 不依赖数据库默认时间，确保结果可预测 +- [ ] 在 `beforeEach` 和 `afterEach` 中正确管理数据库状态 +- [ ] 所有测试都能独立运行且互不干扰 + +#### 🎭 Mock 和外部依赖 + +- [ ] 正确 Mock 了外部依赖服务 (如 FileService、GenerationModel) +- [ ] 在 `beforeEach` 中重置和配置 Mock +- [ ] 验证了 Mock 服务的调用参数和次数 +- [ ] 测试了外部服务错误场景的处理 + +#### 📋 测试覆盖 + +- [ ] 测试覆盖了所有主要方法 (create, query, update, delete) +- [ ] 测试了边界条件和错误场景 +- [ ] 包含了空结果处理的测试 +- [ ] **确认两个环境下的测试结果一致** + +#### 🚨 常见问题检查 + +- [ ] 没有外键约束违反错误 +- [ ] 排序测试结果稳定可预测 +- [ ] Mock 验证无失败 +- [ ] 无测试数据污染问题 + +### 安全警告 ⚠️ + +**数据库 Model 层是安全的第一道防线**。如果 Model 层缺少用户权限检查： + +1. **任何用户都能访问和修改其他用户的数据** +2. **即使上层有权限检查，也可能被绕过** +3. **可能导致严重的数据泄露和安全事故** + +因此，**每个涉及用户数据的 Model 方法都必须包含用户权限检查，且必须有对应的安全测试来验证这些检查的有效性**。 diff --git a/.cursor/rules/testing-guide/electron-ipc-test.mdc b/.cursor/rules/testing-guide/electron-ipc-test.mdc new file mode 100644 index 0000000000..4a91aa1aa1 --- /dev/null +++ b/.cursor/rules/testing-guide/electron-ipc-test.mdc @@ -0,0 +1,80 @@ +--- +description: Electron IPC 接口测试策略 +alwaysApply: false +--- + +### Electron IPC 接口测试策略 🖥️ + +对于涉及 Electron IPC 接口的测试，由于提供真实的 Electron 环境比较复杂，采用 **Mock 返回值** 的方式进行测试。 + +#### 基本 Mock 设置 + +```typescript +import { vi } from 'vitest'; + +import { electronIpcClient } from '@/server/modules/ElectronIPCClient'; + +// Mock Electron IPC 客户端 +vi.mock('@/server/modules/ElectronIPCClient', () => ({ + electronIpcClient: { + getFilePathById: vi.fn(), + deleteFiles: vi.fn(), + // 根据需要添加其他 IPC 方法 + }, +})); +``` + +#### 在测试中设置 Mock 行为 + +```typescript +beforeEach(() => { + // 重置所有 Mock + vi.resetAllMocks(); + + // 设置默认的 Mock 返回值 + vi.mocked(electronIpcClient.getFilePathById).mockResolvedValue('/path/to/file.txt'); + vi.mocked(electronIpcClient.deleteFiles).mockResolvedValue({ + success: true, + }); +}); +``` + +#### 测试不同场景的示例 + +```typescript +it('应该处理文件删除成功的情况', async () => { + // 设置成功场景的 Mock + vi.mocked(electronIpcClient.deleteFiles).mockResolvedValue({ + success: true, + }); + + const result = await service.deleteFiles(['desktop://file1.txt']); + + expect(electronIpcClient.deleteFiles).toHaveBeenCalledWith(['desktop://file1.txt']); + expect(result.success).toBe(true); +}); + +it('应该处理文件删除失败的情况', async () => { + // 设置失败场景的 Mock + vi.mocked(electronIpcClient.deleteFiles).mockRejectedValue(new Error('删除失败')); + + const result = await service.deleteFiles(['desktop://file1.txt']); + + expect(result.success).toBe(false); + expect(result.errors).toBeDefined(); +}); +``` + +#### Mock 策略的优势 + +1. **环境简化**: 避免了复杂的 Electron 环境搭建 +2. **测试可控**: 可以精确控制 IPC 调用的返回值和行为 +3. **场景覆盖**: 容易测试各种成功/失败场景 +4. **执行速度**: Mock 调用比真实 IPC 调用更快 + +#### 注意事项 + +- **Mock 准确性**: 确保 Mock 的行为与真实 IPC 接口行为一致 +- **类型安全**: 使用 `vi.mocked()` 确保类型安全 +- **Mock 重置**: 在 `beforeEach` 中重置 Mock 状态，避免测试间干扰 +- **调用验证**: 不仅要验证返回值，还要验证 IPC 方法是否被正确调用 diff --git a/.cursor/rules/testing-guide/testing-guide.mdc b/.cursor/rules/testing-guide/testing-guide.mdc new file mode 100644 index 0000000000..7140ffa9e3 --- /dev/null +++ b/.cursor/rules/testing-guide/testing-guide.mdc @@ -0,0 +1,401 @@ +--- +globs: *.test.ts,*.test.tsx +alwaysApply: false +--- + +# 测试指南 - LobeChat Testing Guide + +## 🧪 测试环境概览 + +LobeChat 项目使用 Vitest 测试库，配置了两种不同的测试环境： + +### 客户端测试环境 (DOM Environment) + +- **配置文件**: [vitest.config.ts](mdc:vitest.config.ts) +- **环境**: Happy DOM (浏览器环境模拟) +- **数据库**: PGLite (浏览器环境的 PostgreSQL) +- **用途**: 测试前端组件、客户端逻辑、React 组件等 +- **设置文件**: [tests/setup.ts](mdc:tests/setup.ts) + +### 服务端测试环境 (Node Environment) + +- **配置文件**: [vitest.config.server.ts](mdc:vitest.config.server.ts) +- **环境**: Node.js +- **数据库**: 真实的 PostgreSQL 数据库 +- **并发限制**: 单线程运行 (`singleFork: true`) +- **用途**: 测试数据库模型、服务端逻辑、API 端点等 +- **设置文件**: [tests/setup-db.ts](mdc:tests/setup-db.ts) + +## 🚀 测试运行命令 + +### package.json 脚本说明 + +查看 [package.json](mdc:package.json) 中的测试相关脚本： + +```json +{ + "test": "npm run test-app && npm run test-server", + "test-app": "vitest run --config vitest.config.ts", + "test-app:coverage": "vitest run --config vitest.config.ts --coverage", + "test-server": "vitest run --config vitest.config.server.ts", + "test-server:coverage": "vitest run --config vitest.config.server.ts --coverage" +} +``` + +### 推荐的测试运行方式 + +#### ⚠️ 重要提醒 + +**🚨 性能警告**: + +- **永远不要直接运行整个项目的所有测试用例** - 项目包含 3000+ 测试用例，完整运行需要约 10 分钟 +- **务必使用文件过滤或测试名称过滤** - 始终指定具体的测试文件或测试名称模式 +- **避免无意中触发全量测试** - 某些看似针对单个文件的命令实际上会运行所有测试 + +#### ✅ 正确的命令格式 + +```bash +# 运行所有客户端测试 +npx vitest run --config vitest.config.ts + +# 运行所有服务端测试 +npx vitest run --config vitest.config.server.ts + +# 运行特定测试文件 (支持模糊匹配) +npx vitest run --config vitest.config.ts basic +npx vitest run --config vitest.config.ts user.test.ts + +# 运行特定文件的特定行号 +npx vitest run --config vitest.config.ts src/utils/helper.test.ts:25 +npx vitest run --config vitest.config.ts basic/foo.test.ts:10,basic/foo.test.ts:25 + +# 过滤特定测试用例名称 +npx vitest -t "test case name" --config vitest.config.ts + +# 组合使用文件和测试名称过滤 +npx vitest run --config vitest.config.ts filename.test.ts -t "specific test" +``` + +#### ❌ 避免的命令格式 + +```bash +# ❌ 这些命令会运行所有 3000+ 测试用例，耗时约 10 分钟！ +npm test +npm run test +pnpm test +pnpm run test + +# ❌ 这些命令看似针对单个文件，但实际会运行所有测试用例！, 需要直接运行 vitest 命令不要使用 test npm script +npm test src/libs/model-runtime/utils/openaiCompatibleFactory/index.test.ts +pnpm test src/components/Button/index.test.tsx + +# ❌ 不要使用 pnpm test xxx (这不是有效的 vitest 命令) +pnpm test some-file + +# ❌ 不要使用裸 vitest (会进入 watch 模式) +vitest test-file.test.ts + +# ❌ 不要混淆测试环境 +npx vitest run --config vitest.config.server.ts client-component.test.ts +``` + +### 关键运行参数说明 + +- **`vitest run`**: 运行一次测试然后退出 (避免 watch 模式) +- **`vitest`**: 默认进入 watch 模式，持续监听文件变化 +- **`--config`**: 指定配置文件，选择正确的测试环境 +- **`-t`**: 过滤测试用例名称，支持正则表达式 +- **`--coverage`**: 生成测试覆盖率报告 + +## 🔧 测试修复原则 + +### 核心原则 ⚠️ + +1. **充分阅读测试代码**: 在修复测试之前，必须完整理解测试的意图和实现 +2. **测试优先修复**: 如果是测试本身写错了，修改测试而不是实现代码 +3. **专注单一问题**: 只修复指定的测试，不要添加额外测试或功能 +4. **不自作主张**: 不要因为发现其他问题就直接修改，先提出再讨论 + +### 测试修复流程 + +```mermaid +flowchart TD + subgraph "阶段一：分析与复现" + A[开始：收到测试失败报告] --> B[定位并运行失败的测试]; + B --> C{是否能在本地复现?}; + C -->|否| D[检查测试环境/配置/依赖]; + C -->|是| E[分析：阅读测试代码、错误日志、Git 历史]; + end + + subgraph "阶段二：诊断与调试" + E --> F[建立假设：问题出在测试、代码还是环境?]; + F --> G["调试：使用 console.log 或 debugger 深入检查"]; + G --> H{假设是否被证实?}; + H -->|否, 重新假设| F; + end + + subgraph "阶段三：修复与验证" + H -->|是| I{确定根本原因}; + I -->|测试逻辑错误| J[修复测试代码]; + I -->|实现代码 Bug| K[修复实现代码]; + I -->|环境/配置问题| L[修复配置或依赖]; + J --> M[验证修复：重新运行失败的测试]; + K --> M; + L --> M; + M --> N{测试是否通过?}; + N -->|否, 修复无效| F; + N -->|是| O[扩大验证：运行当前文件内所有测试]; + O --> P{是否全部通过?}; + P -->|否, 引入新问题| F; + end + + subgraph "阶段四：总结" + P -->|是| Q[完成：撰写修复总结]; + end + + D --> F; +``` + +### 修复完成后的总结 + +测试修复完成后，应该提供简要说明，包括： + +1. **错误原因分析**: 说明测试失败的根本原因 + - 测试逻辑错误 + - 实现代码bug + - 环境配置问题 + - 依赖变更导致的问题 + +2. **修复方法说明**: 简述采用的修复方式 + - 修改了哪些文件 + - 采用了什么解决方案 + - 为什么选择这种修复方式 + +**示例格式**: + +```markdown +## 测试修复总结 + +**错误原因**: 测试中的 mock 数据格式与实际 API 返回格式不匹配，导致断言失败。 + +**修复方法**: 更新了测试文件中的 mock 数据结构，使其与最新的 API 响应格式保持一致。具体修改了 `user.test.ts` 中的 `mockUserData` 对象结构。 +``` + +## 📂 测试文件组织 + +### 文件命名约定 + +- **客户端测试**: `*.test.ts`, `*.test.tsx` (任意位置) +- **服务端测试**: `src/database/models/**/*.test.ts`, `src/database/server/**/*.test.ts` (限定路径) + +### 测试文件组织风格 + +项目采用 **测试文件与源文件同目录** 的组织风格： + +- 测试文件放在对应源文件的同一目录下 +- 命名格式：`原文件名.test.ts` 或 `原文件名.test.tsx` + +例如： + +``` +src/components/Button/ +├── index.tsx # 源文件 +└── index.test.tsx # 测试文件 +``` + +## 🛠️ 测试调试技巧 + +### 运行失败测试的步骤 + +1. **确定测试类型**: 查看文件路径确定使用哪个配置 +2. **运行单个测试**: 使用 `-t` 参数隔离问题 +3. **检查错误日志**: 仔细阅读错误信息和堆栈跟踪 +4. **查看最近修改记录**: 检查相关文件的最近变更情况 +5. **添加调试日志**: 在测试中添加 `console.log` 了解执行流程 + +### TypeScript 类型处理 📝 + +在测试中，为了提高编写效率和可读性，可以适当放宽 TypeScript 类型检测： + +#### ✅ 推荐的类型放宽策略 + +```typescript +// ✅ 使用非空断言访问测试中确定存在的属性 +const result = await someFunction(); +expect(result!.data).toBeDefined(); +expect(result!.status).toBe('success'); + +// ✅ 使用 any 类型简化复杂的 Mock 设置 +const mockStream = new ReadableStream() as any; +mockStream.toReadableStream = () => mockStream; +``` + +#### 🎯 适用场景 + +- **Mock 对象**: 对于测试用的 Mock 数据，使用 `as any` 避免复杂的类型定义 +- **第三方库**: 处理复杂的第三方库类型时，适当使用 `any` 提高效率 +- **测试断言**: 在确定对象存在的测试场景中，使用 `!` 非空断言 +- **临时调试**: 快速编写测试时，先用 `any` 保证功能，后续可选择性地优化类型 + +#### ⚠️ 注意事项 + +- **适度使用**: 不要过度依赖 `any`，核心业务逻辑的类型仍应保持严格 +- **文档说明**: 对于使用 `any` 的复杂场景，添加注释说明原因 +- **测试覆盖**: 确保即使使用了 `any`，测试仍能有效验证功能正确性 + +### 检查最近修改记录 🔍 + +为了更好地判断测试失败的根本原因，需要**系统性地检查相关文件的修改历史**。这是问题定位的关键步骤。 + +#### 第一步：确定需要检查的文件范围 + +1. **测试文件本身**: `path/to/component.test.ts` +2. **对应的实现文件**: `path/to/component.ts` 或 `path/to/component/index.ts` +3. **相关依赖文件**: 测试或实现中导入的其他模块 + +#### 第二步：检查当前工作目录状态 + +```bash +# 查看所有未提交的修改状态 +git status + +# 重点关注测试文件和实现文件是否有未提交的修改 +git status | grep -E "(test|spec)" +``` + +#### 第三步：检查未提交的修改内容 + +```bash +# 查看测试文件的未提交修改 (工作区 vs 暂存区) +git diff path/to/component.test.ts | cat + +# 查看对应实现文件的未提交修改 +git diff path/to/component.ts | cat + +# 查看已暂存但未提交的修改 +git diff --cached path/to/component.test.ts | cat +git diff --cached path/to/component.ts | cat +``` + +#### 第四步：检查提交历史和时间相关性 + +**首先查看提交时间，判断修改的时效性**： + +```bash +# 查看测试文件的最近提交历史，包含提交时间 +git log --pretty=format:"%h %ad %s" --date=relative -5 path/to/component.test.ts | cat + +# 查看实现文件的最近提交历史，包含提交时间 +git log --pretty=format:"%h %ad %s" --date=relative -5 path/to/component.ts | cat + +# 查看详细的提交时间（ISO格式，便于精确判断） +git log --pretty=format:"%h %ad %an %s" --date=iso -3 path/to/component.ts | cat +git log --pretty=format:"%h %ad %an %s" --date=iso -3 path/to/component.test.ts | cat +``` + +**判断提交的参考价值**： + +1. **最近提交（24小时内）**: 🔴 **高度相关** - 很可能是导致测试失败的直接原因 +2. **近期提交（1-7天内）**: 🟡 **中等相关** - 可能相关，需要仔细分析修改内容 +3. **较早提交（超过1周）**: ⚪ **低相关性** - 除非是重大重构，否则不太可能是直接原因 + +#### 第五步：基于时间相关性查看具体修改内容 + +**根据提交时间的远近，优先查看最近的修改**： + +```bash +# 如果有24小时内的提交，重点查看这些修改 +git show HEAD -- path/to/component.test.ts | cat +git show HEAD -- path/to/component.ts | cat + +# 查看次新的提交（如果最新提交时间较远） +git show HEAD~1 -- path/to/component.ts | cat +git show path/to/component.ts < recent-commit-hash > -- | cat + +# 对比最近两次提交的差异 +git diff HEAD~1 HEAD -- path/to/component.ts | cat +``` + +#### 第六步：分析修改与测试失败的关系 + +基于修改记录和时间相关性判断： + +1. **最近修改了实现代码**: + + ```bash + # 重点检查实现逻辑的变化 + git diff HEAD~1 path/to/component.ts | cat + ``` + + - 很可能是实现代码的变更导致测试失败 + - 检查实现逻辑是否正确 + - 确认测试是否需要相应更新 + +2. **最近修改了测试代码**: + + ```bash + # 重点检查测试逻辑的变化 + git diff HEAD~1 path/to/component.test.ts | cat + ``` + + - 可能是测试本身写错了 + - 检查测试逻辑和断言是否正确 + - 确认测试是否符合实现的预期行为 + +3. **两者都有最近修改**: + + ```bash + # 对比两个文件的修改时间 + git log --pretty=format:"%ad %f" --date=iso -1 path/to/component.ts | cat + git log --pretty=format:"%ad %f" --date=iso -1 path/to/component.test.ts | cat + ``` + + - 需要综合分析两者的修改 + - 确定哪个修改更可能导致问题 + - 优先检查时间更近的修改 + +4. **都没有最近修改**: + - 可能是依赖变更或环境问题 + - 检查 `package.json`、配置文件等的修改 + - 查看是否有全局性的代码重构 + +#### 修改记录检查示例 + +```bash +# 完整的检查流程示例 +echo "=== 检查文件修改状态 ===" +git status | grep component + +echo "=== 检查未提交修改 ===" +git diff src/components/Button/index.test.tsx | cat +git diff src/components/Button/index.tsx | cat + +echo "=== 检查提交历史和时间 ===" +git log --pretty=format:"%h %ad %s" --date=relative -3 src/components/Button/index.test.tsx | cat +git log --pretty=format:"%h %ad %s" --date=relative -3 src/components/Button/index.tsx | cat + +echo "=== 根据时间优先级查看修改内容 ===" +# 如果有24小时内的提交，重点查看 +git show HEAD -- src/components/Button/index.tsx | cat +``` + +## 特殊场景的测试 + +针对一些特殊场景的测试，需要阅读相关文件： + +- [Electron IPC 接口测试策略](mdc:./electron-ipc-test.mdc) +- [数据库 Model 测试指南](mdc:./db-model-test.mdc) + +## 🎯 总结 + +修复测试时，记住以下关键点： + +- **使用正确的命令**: `npx vitest run --config [config-file]` +- **理解测试意图**: 先读懂测试再修复 +- **查看最近修改**: 检查相关文件的 git 修改记录，判断问题根源 +- **选择正确环境**: 客户端测试用 `vitest.config.ts`，服务端用 `vitest.config.server.ts` +- **专注单一问题**: 只修复当前的测试失败 +- **验证修复结果**: 确保修复后测试通过且无副作用 +- **提供修复总结**: 说明错误原因和修复方法 +- **Model 测试安全第一**: 必须包含用户权限检查和对应的安全测试 +- **Model 双环境验证**: 必须在 PGLite 和 PostgreSQL 两个环境下都验证通过 diff --git a/package.json b/package.json index 08a0a0035e..6241ead57b 100644 --- a/package.json +++ b/package.json @@ -326,7 +326,7 @@ "crypto-js": "^4.2.0", "dbdocs": "^0.14.4", "dotenv": "^16.5.0", - "dpdm-fast": "^1.0.7", + "dpdm-fast": "1.0.7", "drizzle-dbml-generator": "^0.10.0", "drizzle-kit": "^0.31.0", "eslint": "^8.57.1", diff --git a/src/config/aiModels/openai.ts b/src/config/aiModels/openai.ts index 621ed11857..0c5480d65e 100644 --- a/src/config/aiModels/openai.ts +++ b/src/config/aiModels/openai.ts @@ -835,11 +835,27 @@ export const openaiSTTModels: AISTTModelCard[] = [ // 图像生成模型 export const openaiImageModels: AIImageModelCard[] = [ + // https://platform.openai.com/docs/models/gpt-image-1 + { + description: 'ChatGPT 原生多模态图片生成模型', + displayName: 'GPT Image 1', + enabled: true, + id: 'gpt-image-1', + parameters: gptImage1ParamsSchema, + type: 'image', + }, { description: '最新的 DALL·E 模型，于2023年11月发布。支持更真实、准确的图像生成，具有更强的细节表现力', displayName: 'DALL·E 3', id: 'dall-e-3', + parameters: { + prompt: { default: '' }, + size: { + default: '1024x1024', + enum: ['1024x1024', '1792x1024', '1024x1792'], + }, + }, pricing: { hd: 0.08, standard: 0.04, @@ -851,21 +867,20 @@ export const openaiImageModels: AIImageModelCard[] = [ description: '第二代 DALL·E 模型，支持更真实、准确的图像生成，分辨率是第一代的4倍', displayName: 'DALL·E 2', id: 'dall-e-2', + parameters: { + imageUrl: { default: null }, + prompt: { default: '' }, + size: { + default: '1024x1024', + enum: ['256x256', '512x512', '1024x1024'], + }, + }, pricing: { input: 0.02, // $0.020 per image (1024×1024) }, resolutions: ['256x256', '512x512', '1024x1024'], type: 'image', }, - // https://platform.openai.com/docs/models/gpt-image-1 - { - description: 'ChatGPT 原生多模态图片生成模型', - displayName: 'GPT Image 1', - enabled: true, - id: 'gpt-image-1', - parameters: gptImage1ParamsSchema, - type: 'image', - }, ]; // GPT-4o 和 GPT-4o-mini 实时模型 diff --git a/src/libs/model-runtime/BaseAI.ts b/src/libs/model-runtime/BaseAI.ts index adcf73d756..b66e316625 100644 --- a/src/libs/model-runtime/BaseAI.ts +++ b/src/libs/model-runtime/BaseAI.ts @@ -43,6 +43,7 @@ export abstract class LobeOpenAICompatibleRuntime { abstract client: OpenAI; abstract chat(payload: ChatStreamPayload, options?: ChatMethodOptions): Promise; + abstract createImage(payload: CreateImagePayload): Promise; abstract models(): Promise; diff --git a/src/libs/model-runtime/hunyuan/index.ts b/src/libs/model-runtime/hunyuan/index.ts index 0b2106088d..8379a3046e 100644 --- a/src/libs/model-runtime/hunyuan/index.ts +++ b/src/libs/model-runtime/hunyuan/index.ts @@ -12,7 +12,8 @@ export const LobeHunyuanAI = createOpenAICompatibleRuntime({ chatCompletion: { handlePayload: (payload) => { // eslint-disable-next-line unused-imports/no-unused-vars, @typescript-eslint/no-unused-vars - const { enabledSearch, frequency_penalty, model, presence_penalty, thinking, ...rest } = payload; + const { enabledSearch, frequency_penalty, model, presence_penalty, thinking, ...rest } = + payload; return { ...rest, @@ -30,11 +31,8 @@ export const LobeHunyuanAI = createOpenAICompatibleRuntime({ search_info: true, }), ...(model === 'hunyuan-a13b' && { - enable_thinking: thinking?.type === 'enabled' - ? true - : thinking?.type === 'disabled' - ? false - : undefined + enable_thinking: + thinking?.type === 'enabled' ? true : thinking?.type === 'disabled' ? false : undefined, }), } as any; }, diff --git a/src/libs/model-runtime/novita/__snapshots__/index.test.ts.snap b/src/libs/model-runtime/novita/__snapshots__/index.test.ts.snap index bd41ed3610..afe2e4e50d 100644 --- a/src/libs/model-runtime/novita/__snapshots__/index.test.ts.snap +++ b/src/libs/model-runtime/novita/__snapshots__/index.test.ts.snap @@ -11,6 +11,7 @@ exports[`NovitaAI > models > should get models 1`] = ` "id": "meta-llama/llama-3-8b-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -22,6 +23,7 @@ exports[`NovitaAI > models > should get models 1`] = ` "id": "meta-llama/llama-3-70b-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -33,6 +35,7 @@ exports[`NovitaAI > models > should get models 1`] = ` "id": "meta-llama/llama-3.1-8b-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -44,6 +47,7 @@ exports[`NovitaAI > models > should get models 1`] = ` "id": "meta-llama/llama-3.1-70b-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -55,6 +59,7 @@ exports[`NovitaAI > models > should get models 1`] = ` "id": "meta-llama/llama-3.1-405b-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -67,6 +72,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "google/gemma-2-9b-it", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -78,6 +84,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "jondurbin/airoboros-l2-70b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -89,6 +96,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "nousresearch/hermes-2-pro-llama-3-8b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -100,6 +108,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "mistralai/mistral-7b-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -111,6 +120,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "cognitivecomputations/dolphin-mixtral-8x22b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -122,6 +132,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "sao10k/l3-70b-euryale-v2.1", "maxOutput": undefined, "reasoning": true, + "type": "chat", "vision": false, }, { @@ -133,6 +144,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "sophosympatheia/midnight-rose-70b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -144,6 +156,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "gryphe/mythomax-l2-13b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -155,6 +168,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "nousresearch/nous-hermes-llama2-13b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -166,6 +180,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "Nous-Hermes-2-Mixtral-8x7B-DPO", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -177,6 +192,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "lzlv_70b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -188,6 +204,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "teknium/openhermes-2.5-mistral-7b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -199,6 +216,7 @@ Designed for a wide variety of tasks, it empowers developers and researchers to "id": "microsoft/wizardlm-2-8x22b", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, ] diff --git a/src/libs/model-runtime/openai/__snapshots__/index.test.ts.snap b/src/libs/model-runtime/openai/__snapshots__/index.test.ts.snap index ddbc5eaed9..a44a1655aa 100644 --- a/src/libs/model-runtime/openai/__snapshots__/index.test.ts.snap +++ b/src/libs/model-runtime/openai/__snapshots__/index.test.ts.snap @@ -10,6 +10,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "whisper-1", "maxOutput": undefined, "reasoning": false, + "type": "stt", "vision": false, }, { @@ -20,6 +21,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "davinci-002", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -30,6 +32,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -40,6 +43,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "dall-e-2", "maxOutput": undefined, "reasoning": false, + "type": "image", "vision": false, }, { @@ -50,6 +54,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-16k", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -60,6 +65,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "tts-1-hd-1106", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -70,6 +76,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "tts-1-hd", "maxOutput": undefined, "reasoning": false, + "type": "tts", "vision": false, }, { @@ -80,6 +87,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-16k-0613", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -90,6 +98,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "text-embedding-3-large", "maxOutput": undefined, "reasoning": false, + "type": "embedding", "vision": false, }, { @@ -100,6 +109,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4-1106-vision-preview", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -110,6 +120,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-instruct-0914", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -120,6 +131,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4-0125-preview", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -130,6 +142,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4-turbo-preview", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -140,6 +153,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-instruct", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -150,6 +164,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-0301", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -160,6 +175,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-0613", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -170,6 +186,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "tts-1", "maxOutput": undefined, "reasoning": false, + "type": "tts", "vision": false, }, { @@ -180,6 +197,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "dall-e-3", "maxOutput": undefined, "reasoning": false, + "type": "image", "vision": false, }, { @@ -190,6 +208,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-1106", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -200,6 +219,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4-1106-preview", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -210,6 +230,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "babbage-002", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -220,6 +241,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "tts-1-1106", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -230,6 +252,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4-vision-preview", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": true, }, { @@ -240,6 +263,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "text-embedding-3-small", "maxOutput": undefined, "reasoning": false, + "type": "embedding", "vision": false, }, { @@ -250,6 +274,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4", "maxOutput": 4096, "reasoning": false, + "type": "chat", "vision": true, }, { @@ -260,6 +285,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "text-embedding-ada-002", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -270,6 +296,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-3.5-turbo-0125", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, { @@ -280,6 +307,7 @@ exports[`LobeOpenAI > models > should get models 1`] = ` "id": "gpt-4-0613", "maxOutput": undefined, "reasoning": false, + "type": "chat", "vision": false, }, ] diff --git a/src/libs/model-runtime/openai/index.test.ts b/src/libs/model-runtime/openai/index.test.ts index ee3369203a..793f95d050 100644 --- a/src/libs/model-runtime/openai/index.test.ts +++ b/src/libs/model-runtime/openai/index.test.ts @@ -4,7 +4,6 @@ import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; // 引入模块以便于对函数进行spy import { ChatStreamCallbacks } from '@/libs/model-runtime'; -import * as openai from '@/libs/model-runtime/openai'; import * as debugStreamModule from '../utils/debugStream'; import officalOpenAIModels from './fixtures/openai-models.json'; @@ -13,22 +12,13 @@ import { LobeOpenAI } from './index'; // Mock the console.error to avoid polluting test output vi.spyOn(console, 'error').mockImplementation(() => {}); -// Mock fetch for most tests, but will be restored for convertImageUrlToFile tests +// Mock fetch for most tests, but will be restored for real network tests const mockFetch = vi.fn(); global.fetch = mockFetch; -const convertImageUrlToFileSpy = vi.spyOn(openai, 'convertImageUrlToFile'); - describe('LobeOpenAI', () => { let instance: InstanceType; - // Create mock params for createImage tests - only gpt-image-1 supported params - const mockParams = { - prompt: 'test prompt', - imageUrls: [] as string[], - size: '1024x1024' as const, - }; - beforeEach(() => { instance = new LobeOpenAI({ apiKey: 'test' }); @@ -40,27 +30,6 @@ describe('LobeOpenAI', () => { // Mock responses.create for responses API tests vi.spyOn(instance['client'].responses, 'create').mockResolvedValue(new ReadableStream() as any); - - // Mock convertImageUrlToFile to return a mock File object - convertImageUrlToFileSpy.mockResolvedValue({ - name: 'image.png', - type: 'image/png', - size: 1024, - } as any); - - // Mock fetch response for most tests - mockFetch.mockResolvedValue({ - ok: true, - arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)), - headers: { - get: (header: string) => { - if (header === 'content-type') { - return 'image/png'; - } - return null; - }, - }, - }); }); afterEach(() => { @@ -282,312 +251,6 @@ describe('LobeOpenAI', () => { }); }); - describe('createImage', () => { - it('should generate an image with gpt-image-1', async () => { - // Arrange - const mockResponse = { data: [{ b64_json: 'test-base64-string' }] }; - const generateSpy = vi - .spyOn(instance['client'].images, 'generate') - .mockResolvedValue(mockResponse as any); - - // Act - const result = await instance.createImage({ - model: 'gpt-image-1', - params: { - ...mockParams, - prompt: 'A cute cat', - size: '1024x1024', - }, - }); - - // Assert - expect(generateSpy).toHaveBeenCalledWith( - expect.objectContaining({ - model: 'gpt-image-1', - prompt: 'A cute cat', - n: 1, - size: '1024x1024', - }), - ); - expect(result.imageUrl).toBe('data:image/png;base64,test-base64-string'); - }); - - it('should edit an image from a URL', async () => { - // Arrange - const mockResponse = { data: [{ b64_json: 'edited-base64-string' }] }; - const editSpy = vi - .spyOn(instance['client'].images, 'edit') - .mockResolvedValue(mockResponse as any); - - // Temporarily restore the spy to use real implementation - convertImageUrlToFileSpy.mockRestore(); - - const imageUrl = 'https://lobehub.com/_next/static/media/logo.98482105.png'; - - // Act - const result = await instance.createImage({ - model: 'gpt-image-1', - params: { - ...mockParams, - prompt: 'A cat in a hat', - imageUrls: [imageUrl], - }, - }); - - // Assert - expect(editSpy).toHaveBeenCalled(); - const callArg = editSpy.mock.calls[0][0]; - expect(callArg.model).toBe('gpt-image-1'); - expect(callArg.prompt).toBe('A cat in a hat'); - expect(result.imageUrl).toBe('data:image/png;base64,edited-base64-string'); - - // Restore the spy for other tests - convertImageUrlToFileSpy.mockResolvedValue({ - name: 'image.png', - type: 'image/png', - size: 1024, - } as any); - }); - - it('should handle `size` set to `auto`', async () => { - // Arrange - const mockResponse = { data: [{ b64_json: 'test-base64-string' }] }; - const generateSpy = vi - .spyOn(instance['client'].images, 'generate') - .mockResolvedValue(mockResponse as any); - - // Act - await instance.createImage({ - model: 'gpt-image-1', - params: { - ...mockParams, - prompt: 'A cute cat', - size: 'auto', - }, - }); - - // Assert - expect(generateSpy).toHaveBeenCalledWith( - expect.objectContaining({ - model: 'gpt-image-1', - prompt: 'A cute cat', - n: 1, - }), - ); - // Should not include size when it's 'auto' - expect(generateSpy.mock.calls[0][0]).not.toHaveProperty('size'); - }); - - it('should throw an error if convertImageUrlToFile fails', async () => { - // Arrange - const imageUrl = 'https://example.com/test-image.png'; - - // Mock fetch to fail for the image URL, which will cause convertImageUrlToFile to fail - vi.mocked(global.fetch).mockRejectedValueOnce(new Error('Network error')); - - // Mock the OpenAI API methods to ensure they don't get called - const generateSpy = vi.spyOn(instance['client'].images, 'generate'); - const editSpy = vi.spyOn(instance['client'].images, 'edit'); - - // Act & Assert - Note: imageUrls must be non-empty array to trigger isImageEdit = true - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { - prompt: 'A cat in a hat', - imageUrls: [imageUrl], // This is the key - non-empty array - }, - }), - ).rejects.toThrow('Failed to convert image URLs to File objects: Error: Network error'); - - // Verify that OpenAI API methods were not called since conversion failed - expect(generateSpy).not.toHaveBeenCalled(); - expect(editSpy).not.toHaveBeenCalled(); - }); - - it('should throw an error when image response is missing data array', async () => { - // Arrange - const mockInvalidResponse = {}; // missing data property - vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockInvalidResponse as any); - - // Act & Assert - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { ...mockParams, prompt: 'A cute cat' }, - }), - ).rejects.toThrow('Invalid image response: missing or empty data array'); - }); - - it('should throw an error when image response data array is empty', async () => { - // Arrange - const mockInvalidResponse = { data: [] }; // empty data array - vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockInvalidResponse as any); - - // Act & Assert - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { ...mockParams, prompt: 'A cute cat' }, - }), - ).rejects.toThrow('Invalid image response: missing or empty data array'); - }); - - it('should throw an error when first data item is null', async () => { - // Arrange - const mockInvalidResponse = { data: [null] }; // first item is null - vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockInvalidResponse as any); - - // Act & Assert - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { ...mockParams, prompt: 'A cute cat' }, - }), - ).rejects.toThrow('Invalid image response: first data item is null or undefined'); - }); - - it('should throw an error when first data item is undefined', async () => { - // Arrange - const mockInvalidResponse = { data: [undefined] }; // first item is undefined - vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockInvalidResponse as any); - - // Act & Assert - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { ...mockParams, prompt: 'A cute cat' }, - }), - ).rejects.toThrow('Invalid image response: first data item is null or undefined'); - }); - - it('should re-throw OpenAI API errors during image generation', async () => { - // Arrange - const apiError = new OpenAI.APIError( - 400, - { error: { message: 'Bad Request' } }, - 'Error message', - {}, - ); - vi.spyOn(instance['client'].images, 'generate').mockRejectedValue(apiError); - - // Act & Assert - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { ...mockParams, prompt: 'A cute cat' }, - }), - ).rejects.toThrow(apiError); - }); - - it('should throw an error for invalid image response', async () => { - // Arrange - const mockInvalidResponse = { data: [{ url: 'some_url' }] }; // missing b64_json - vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockInvalidResponse as any); - - // Act & Assert - await expect( - instance.createImage({ - model: 'gpt-image-1', - params: { ...mockParams, prompt: 'A cute cat' }, - }), - ).rejects.toThrow('Invalid image response: missing b64_json field'); - }); - }); - - describe('convertImageUrlToFile', () => { - beforeEach(() => { - // Reset the spy to use the real implementation for these tests - convertImageUrlToFileSpy.mockRestore(); - }); - - afterEach(() => { - // Restore the spy for other tests - convertImageUrlToFileSpy.mockResolvedValue({ - name: 'image.png', - type: 'image/png', - size: 1024, - } as any); - }); - - it('should convert the real lobehub logo URL to a FileLike object', async () => { - const imageUrl = 'https://lobehub.com/_next/static/media/logo.98482105.png'; - const file = await openai.convertImageUrlToFile(imageUrl); - - expect(file).toBeDefined(); - expect((file as any).name).toBe('image.png'); - expect((file as any).type).toMatch(/^image\//); - expect((file as any).size).toBeGreaterThan(0); - }); - - it('should convert a base64 data URL to a FileLike object', async () => { - const dataUrl = - 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAAA//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AN//Z'; - const file = await openai.convertImageUrlToFile(dataUrl); - - expect(file).toBeDefined(); - expect((file as any).name).toBe('image.jpeg'); - expect((file as any).type).toBe('image/jpeg'); - }); - - it('should handle different image mime types from data URL', async () => { - const webpDataUrl = 'data:image/webp;base64,UklGRhoAAABXRUJQVlA4TA0AAAAvAAAAEAcQERGIiP4HAA=='; - const file = await openai.convertImageUrlToFile(webpDataUrl); - - expect(file).toBeDefined(); - expect((file as any).name).toBe('image.webp'); - expect((file as any).type).toBe('image/webp'); - }); - }); - - // Separate describe block for mocked fetch scenarios - describe('convertImageUrlToFile - mocked scenarios', () => { - beforeEach(() => { - // Reset the spy to use the real implementation - convertImageUrlToFileSpy.mockRestore(); - }); - - afterEach(() => { - // Restore the spy for other tests - convertImageUrlToFileSpy.mockResolvedValue({ - name: 'image.png', - type: 'image/png', - size: 1024, - } as any); - }); - - it('should throw an error if fetching an image from a URL fails', async () => { - // Use vi.mocked for type-safe mocking - vi.mocked(global.fetch).mockResolvedValueOnce({ - ok: false, - statusText: 'Not Found', - } as any); - - const imageUrl = 'https://example.com/invalid-image.png'; - - await expect(openai.convertImageUrlToFile(imageUrl)).rejects.toThrow( - 'Failed to fetch image from https://example.com/invalid-image.png: Not Found', - ); - }); - - it('should use a default mime type of image/png if content-type header is not available', async () => { - // Use vi.mocked for type-safe mocking - vi.mocked(global.fetch).mockResolvedValueOnce({ - ok: true, - arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)), - headers: { - get: () => null, - }, - } as any); - - const imageUrl = 'https://example.com/image-no-content-type'; - const file = await openai.convertImageUrlToFile(imageUrl); - - expect(file).toBeDefined(); - expect((file as any).type).toBe('image/png'); - }); - }); - describe('responses.handlePayload', () => { it('should add web_search_preview tool when enabledSearch is true', async () => { const payload = { diff --git a/src/libs/model-runtime/openai/index.ts b/src/libs/model-runtime/openai/index.ts index fa0243ae8c..fe18c2e443 100644 --- a/src/libs/model-runtime/openai/index.ts +++ b/src/libs/model-runtime/openai/index.ts @@ -1,9 +1,4 @@ -import debug from 'debug'; -import { toFile } from 'openai'; -import { FileLike } from 'openai/uploads'; - import { responsesAPIModels } from '@/const/models'; -import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/meta-schema'; import { ChatStreamPayload, ModelProvider } from '../types'; import { processMultiProviderModelList } from '../utils/modelParse'; @@ -15,45 +10,8 @@ export interface OpenAIModelCard { } const prunePrefixes = ['o1', 'o3', 'o4', 'codex', 'computer-use']; - const oaiSearchContextSize = process.env.OPENAI_SEARCH_CONTEXT_SIZE; // low, medium, high -const log = debug('lobe-image:openai'); - -/** - * 将图片 URL 转换为 File 对象 - * @param imageUrl - 图片 URL（可以是 HTTP URL 或 base64 data URL） - * @returns FileLike 对象 - */ -export const convertImageUrlToFile = async (imageUrl: string): Promise => { - log('Converting image URL to File: %s', imageUrl.startsWith('data:') ? 'base64 data' : imageUrl); - - let buffer: Buffer; - let mimeType: string; - - if (imageUrl.startsWith('data:')) { - // 处理 base64 data URL - log('Processing base64 image data'); - const [mimeTypePart, base64Data] = imageUrl.split(','); - mimeType = mimeTypePart.split(':')[1].split(';')[0]; - buffer = Buffer.from(base64Data, 'base64'); - } else { - // 处理 HTTP URL - log('Fetching image from URL: %s', imageUrl); - const response = await fetch(imageUrl); - if (!response.ok) { - throw new Error(`Failed to fetch image from ${imageUrl}: ${response.statusText}`); - } - buffer = Buffer.from(await response.arrayBuffer()); - mimeType = response.headers.get('content-type') || 'image/png'; - } - - log('Successfully converted image to buffer, size: %s, mimeType: %s', buffer.length, mimeType); - - // 使用 OpenAI 的 toFile 方法创建 File 对象 - return toFile(buffer, `image.${mimeType.split('/')[1]}`, { type: mimeType }); -}; - export const LobeOpenAI = createOpenAICompatibleRuntime({ baseURL: 'https://api.openai.com/v1', chatCompletion: { @@ -88,91 +46,6 @@ export const LobeOpenAI = createOpenAICompatibleRuntime({ return { ...rest, model, stream: payload.stream ?? true }; }, }, - createImage: async (payload) => { - const { model, params, client } = payload; - log('Creating image with model: %s and params: %O', model, params); - - const defaultInput = { - n: 1, - }; - - // 映射参数名称，将 imageUrls 映射为 image - const paramsMap = new Map([['imageUrls', 'image']]); - const userInput: Record = Object.fromEntries( - Object.entries(params).map(([key, value]) => [ - paramsMap.get(key as RuntimeImageGenParamsValue) ?? key, - value, - ]), - ); - - const isImageEdit = Array.isArray(userInput.image) && userInput.image.length > 0; - // 如果有 imageUrls 参数，将其转换为 File 对象 - if (isImageEdit) { - log('Converting imageUrls to File objects: %O', userInput.image); - try { - // 转换所有图片 URL 为 File 对象 - const imageFiles = await Promise.all( - userInput.image.map((url: string) => convertImageUrlToFile(url)), - ); - - log('Successfully converted %d images to File objects', imageFiles.length); - - // 根据官方文档，如果有多个图片，传递数组；如果只有一个，传递单个 File - userInput.image = imageFiles.length === 1 ? imageFiles[0] : imageFiles; - } catch (error) { - log('Error converting imageUrls to File objects: %O', error); - throw new Error(`Failed to convert image URLs to File objects: ${error}`); - } - } else { - delete userInput.image; - } - - if (userInput.size === 'auto') { - delete userInput.size; - } - - const options = { - model, - ...defaultInput, - ...(userInput as any), - }; - - log('options: %O', options); - - // 判断是否为图片编辑操作 - const img = isImageEdit - ? await client.images.edit(options) - : await client.images.generate(options); - - // 检查响应数据的完整性 - if (!img || !img.data || !Array.isArray(img.data) || img.data.length === 0) { - log('Invalid image response: missing data array'); - throw new Error('Invalid image response: missing or empty data array'); - } - - const imageData = img.data[0]; - if (!imageData) { - log('Invalid image response: first data item is null/undefined'); - throw new Error('Invalid image response: first data item is null or undefined'); - } - - if (!imageData.b64_json) { - log('Invalid image response: missing b64_json field'); - throw new Error('Invalid image response: missing b64_json field'); - } - - // 确定图片的 MIME 类型，默认为 PNG - const mimeType = 'image/png'; // OpenAI 图片生成默认返回 PNG 格式 - - // 将 base64 字符串转换为完整的 data URL - const dataUrl = `data:${mimeType};base64,${imageData.b64_json}`; - - log('Successfully converted base64 to data URL, length: %d', dataUrl.length); - - return { - imageUrl: dataUrl, - }; - }, debug: { chatCompletion: () => process.env.DEBUG_OPENAI_CHAT_COMPLETION === '1', responses: () => process.env.DEBUG_OPENAI_RESPONSES === '1', diff --git a/src/libs/model-runtime/openrouter/__snapshots__/index.test.ts.snap b/src/libs/model-runtime/openrouter/__snapshots__/index.test.ts.snap index e7da6f2cbd..7753fd65c3 100644 --- a/src/libs/model-runtime/openrouter/__snapshots__/index.test.ts.snap +++ b/src/libs/model-runtime/openrouter/__snapshots__/index.test.ts.snap @@ -21,6 +21,7 @@ _These are free, rate-limited endpoints for [Reflection 70B](/models/mattshumer/ }, "reasoning": true, "releasedAt": "2024-09-06", + "type": "chat", "vision": false, }, ] @@ -47,6 +48,7 @@ _These are free, rate-limited endpoints for [Reflection 70B](/models/mattshumer/ }, "reasoning": false, "releasedAt": "2024-09-06", + "type": "chat", "vision": false, }, ] @@ -73,6 +75,7 @@ _These are free, rate-limited endpoints for [Reflection 70B](/models/mattshumer/ }, "reasoning": false, "releasedAt": "2024-09-06", + "type": "chat", "vision": false, }, ] diff --git a/src/libs/model-runtime/ppio/__snapshots__/index.test.ts.snap b/src/libs/model-runtime/ppio/__snapshots__/index.test.ts.snap index 792ecfd3da..aee1b20b3e 100644 --- a/src/libs/model-runtime/ppio/__snapshots__/index.test.ts.snap +++ b/src/libs/model-runtime/ppio/__snapshots__/index.test.ts.snap @@ -10,6 +10,7 @@ exports[`PPIO > models > should get models 1`] = ` "functionCall": false, "id": "deepseek/deepseek-r1/community", "reasoning": true, + "type": "chat", "vision": false, }, { @@ -20,6 +21,7 @@ exports[`PPIO > models > should get models 1`] = ` "functionCall": false, "id": "deepseek/deepseek-v3/community", "reasoning": false, + "type": "chat", "vision": false, }, ] diff --git a/src/libs/model-runtime/utils/modelParse.ts b/src/libs/model-runtime/utils/modelParse.ts index 832a45a7f5..587d3acb55 100644 --- a/src/libs/model-runtime/utils/modelParse.ts +++ b/src/libs/model-runtime/utils/modelParse.ts @@ -136,6 +136,7 @@ const processModelCard = ( reasoningKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)) || knownModel?.abilities?.reasoning || false, + type: model.type || knownModel?.type || 'chat', vision: (visionKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)) && !isExcludedModel) || diff --git a/src/libs/model-runtime/utils/openaiCompatibleFactory/index.test.ts b/src/libs/model-runtime/utils/openaiCompatibleFactory/index.test.ts index b12426ec42..9f372271eb 100644 --- a/src/libs/model-runtime/utils/openaiCompatibleFactory/index.test.ts +++ b/src/libs/model-runtime/utils/openaiCompatibleFactory/index.test.ts @@ -14,6 +14,7 @@ import officalOpenAIModels from '@/libs/model-runtime/openai/fixtures/openai-mod import { sleep } from '@/utils/sleep'; import * as debugStreamModule from '../debugStream'; +import * as openaiHelpers from '../openaiHelpers'; import { createOpenAICompatibleRuntime } from './index'; const provider = 'groq'; @@ -978,6 +979,329 @@ describe('LobeOpenAICompatibleFactory', () => { }); }); + describe('createImage', () => { + beforeEach(() => { + // Mock convertImageUrlToFile since it's already tested in openaiHelpers.test.ts + vi.spyOn(openaiHelpers, 'convertImageUrlToFile').mockResolvedValue( + new File(['mock-file-content'], 'test-image.jpg', { type: 'image/jpeg' }), + ); + }); + + describe('basic image generation', () => { + it('should generate image successfully without imageUrls', async () => { + const mockResponse = { + data: [ + { + b64_json: + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', + }, + ], + }; + + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'dall-e-3', + params: { + prompt: 'A beautiful sunset', + size: '1024x1024', + quality: 'standard', + }, + }; + + const result = await (instance as any).createImage(payload); + + expect(instance['client'].images.generate).toHaveBeenCalledWith({ + model: 'dall-e-3', + n: 1, + prompt: 'A beautiful sunset', + size: '1024x1024', + quality: 'standard', + response_format: 'b64_json', + }); + + expect(result).toEqual({ + imageUrl: + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', + }); + }); + + it('should handle size auto parameter correctly', async () => { + const mockResponse = { + data: [{ b64_json: 'mock-base64-data' }], + }; + + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'dall-e-3', + params: { + prompt: 'A beautiful sunset', + size: 'auto', + }, + }; + + await (instance as any).createImage(payload); + + // size: 'auto' should be removed from the options + expect(instance['client'].images.generate).toHaveBeenCalledWith({ + model: 'dall-e-3', + n: 1, + prompt: 'A beautiful sunset', + response_format: 'b64_json', + }); + }); + + it('should not add response_format parameter for gpt-image-1 model', async () => { + const mockResponse = { + data: [{ b64_json: 'gpt-image-1-base64-data' }], + }; + + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'gpt-image-1', + params: { + prompt: 'A modern digital artwork', + size: '1024x1024', + }, + }; + + const result = await (instance as any).createImage(payload); + + // gpt-image-1 model should not include response_format parameter + expect(instance['client'].images.generate).toHaveBeenCalledWith({ + model: 'gpt-image-1', + n: 1, + prompt: 'A modern digital artwork', + size: '1024x1024', + }); + + expect(result).toEqual({ + imageUrl: 'data:image/png;base64,gpt-image-1-base64-data', + }); + }); + }); + + describe('image editing', () => { + it('should edit image with single imageUrl', async () => { + const mockResponse = { + data: [{ b64_json: 'edited-image-base64' }], + }; + + vi.spyOn(instance['client'].images, 'edit').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'dall-e-2', + params: { + prompt: 'Add a rainbow to this image', + imageUrls: ['https://example.com/image1.jpg'], + mask: 'https://example.com/mask.jpg', + }, + }; + + const result = await (instance as any).createImage(payload); + + expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledWith( + 'https://example.com/image1.jpg', + ); + expect(instance['client'].images.edit).toHaveBeenCalledWith({ + model: 'dall-e-2', + n: 1, + prompt: 'Add a rainbow to this image', + image: expect.any(File), + mask: 'https://example.com/mask.jpg', + response_format: 'b64_json', + }); + + expect(result).toEqual({ + imageUrl: 'data:image/png;base64,edited-image-base64', + }); + }); + + it('should edit image with multiple imageUrls', async () => { + const mockResponse = { + data: [{ b64_json: 'edited-multiple-images-base64' }], + }; + + const mockFile1 = new File(['content1'], 'image1.jpg', { type: 'image/jpeg' }); + const mockFile2 = new File(['content2'], 'image2.jpg', { type: 'image/jpeg' }); + + vi.mocked(openaiHelpers.convertImageUrlToFile) + .mockResolvedValueOnce(mockFile1) + .mockResolvedValueOnce(mockFile2); + + vi.spyOn(instance['client'].images, 'edit').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'dall-e-2', + params: { + prompt: 'Merge these images', + imageUrls: ['https://example.com/image1.jpg', 'https://example.com/image2.jpg'], + }, + }; + + const result = await (instance as any).createImage(payload); + + expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledTimes(2); + expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledWith( + 'https://example.com/image1.jpg', + ); + expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledWith( + 'https://example.com/image2.jpg', + ); + + expect(instance['client'].images.edit).toHaveBeenCalledWith({ + model: 'dall-e-2', + n: 1, + prompt: 'Merge these images', + image: [mockFile1, mockFile2], + response_format: 'b64_json', + }); + + expect(result).toEqual({ + imageUrl: 'data:image/png;base64,edited-multiple-images-base64', + }); + }); + + it('should handle convertImageUrlToFile error', async () => { + vi.mocked(openaiHelpers.convertImageUrlToFile).mockRejectedValue( + new Error('Failed to download image'), + ); + + const payload = { + model: 'dall-e-2', + params: { + prompt: 'Edit this image', + imageUrls: ['https://invalid-url.com/image.jpg'], + }, + }; + + await expect((instance as any).createImage(payload)).rejects.toThrow( + 'Failed to convert image URLs to File objects: Error: Failed to download image', + ); + }); + }); + + describe('error handling', () => { + it('should throw error when API response is invalid - no data', async () => { + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({} as any); + + const payload = { + model: 'dall-e-3', + params: { prompt: 'Test prompt' }, + }; + + await expect((instance as any).createImage(payload)).rejects.toThrow( + 'Invalid image response: missing or empty data array', + ); + }); + + it('should throw error when API response is invalid - empty data array', async () => { + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({ + data: [], + } as any); + + const payload = { + model: 'dall-e-3', + params: { prompt: 'Test prompt' }, + }; + + await expect((instance as any).createImage(payload)).rejects.toThrow( + 'Invalid image response: missing or empty data array', + ); + }); + + it('should throw error when first data item is null', async () => { + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({ + data: [null], + } as any); + + const payload = { + model: 'dall-e-3', + params: { prompt: 'Test prompt' }, + }; + + await expect((instance as any).createImage(payload)).rejects.toThrow( + 'Invalid image response: first data item is null or undefined', + ); + }); + + it('should throw error when b64_json is missing', async () => { + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({ + data: [{ url: 'https://example.com/image.jpg' }], + } as any); + + const payload = { + model: 'dall-e-3', + params: { prompt: 'Test prompt' }, + }; + + await expect((instance as any).createImage(payload)).rejects.toThrow( + 'Invalid image response: missing b64_json field', + ); + }); + }); + + describe('parameter mapping', () => { + it('should map imageUrls parameter to image', async () => { + const mockResponse = { + data: [{ b64_json: 'test-base64' }], + }; + + vi.spyOn(instance['client'].images, 'edit').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'dall-e-2', + params: { + prompt: 'Test prompt', + imageUrls: ['https://example.com/image.jpg'], + customParam: 'should remain unchanged', + }, + }; + + await (instance as any).createImage(payload); + + expect(instance['client'].images.edit).toHaveBeenCalledWith({ + model: 'dall-e-2', + n: 1, + prompt: 'Test prompt', + image: expect.any(File), + customParam: 'should remain unchanged', + response_format: 'b64_json', + }); + }); + + it('should handle parameters without imageUrls', async () => { + const mockResponse = { + data: [{ b64_json: 'test-base64' }], + }; + + vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any); + + const payload = { + model: 'dall-e-3', + params: { + prompt: 'Test prompt', + quality: 'hd', + style: 'vivid', + }, + }; + + await (instance as any).createImage(payload); + + expect(instance['client'].images.generate).toHaveBeenCalledWith({ + model: 'dall-e-3', + n: 1, + prompt: 'Test prompt', + quality: 'hd', + style: 'vivid', + response_format: 'b64_json', + }); + }); + }); + }); + describe('models', () => { it('should get models with third party model list', async () => { vi.spyOn(instance['client'].models, 'list').mockResolvedValue({ @@ -993,54 +1317,82 @@ describe('LobeOpenAICompatibleFactory', () => { expect(list).toEqual([ { + abilities: { + functionCall: true, + vision: true, + }, + config: { + deploymentName: 'gpt-4o', + }, contextWindowTokens: 128000, - releasedAt: '2023-10-25', description: 'ChatGPT-4o 是一款动态模型，实时更新以保持当前最新版本。它结合了强大的语言理解与生成能力，适合于大规模应用场景，包括客户服务、教育和技术支持。', displayName: 'GPT-4o', enabled: true, - functionCall: true, id: 'gpt-4o', + maxOutput: 4096, pricing: { + cachedInput: 1.25, input: 2.5, output: 10, }, - vision: true, + providerId: 'azure', + releasedAt: '2024-05-13', + source: 'builtin', + type: 'chat', }, { + abilities: { + functionCall: true, + vision: true, + }, contextWindowTokens: 200000, description: 'Claude 3 Haiku 是 Anthropic 的最快且最紧凑的模型，旨在实现近乎即时的响应。它具有快速且准确的定向性能。', displayName: 'Claude 3 Haiku', - functionCall: true, + enabled: false, id: 'claude-3-haiku-20240307', maxOutput: 4096, pricing: { input: 0.25, output: 1.25, }, + providerId: 'anthropic', releasedAt: '2024-03-07', - vision: true, + settings: { + extendParams: ['disableContextCaching'], + }, + source: 'builtin', + type: 'chat', }, { + abilities: { + functionCall: true, + vision: true, + }, + config: { + deploymentName: 'gpt-4o-mini', + }, contextWindowTokens: 128000, - description: - 'GPT-4o mini是OpenAI在GPT-4 Omni之后推出的最新模型，支持图文输入并输出文本。作为他们最先进的小型模型，它比其他近期的前沿模型便宜很多，并且比GPT-3.5 Turbo便宜超过60%。它保持了最先进的智能，同时具有显著的性价比。GPT-4o mini在MMLU测试中获得了 82% 的得分，目前在聊天偏好上排名高于 GPT-4。', - displayName: 'GPT-4o mini', - enabled: true, - functionCall: true, + description: 'GPT-4o Mini，小型高效模型，具备与GPT-4o相似的卓越性能。', + displayName: 'GPT 4o Mini', + enabled: false, id: 'gpt-4o-mini', - maxOutput: 16385, + maxOutput: 4096, pricing: { + cachedInput: 0.075, input: 0.15, output: 0.6, }, + providerId: 'azure', releasedAt: '2023-10-26', - vision: true, + source: 'builtin', + type: 'chat', }, { id: 'gemini', releasedAt: '2025-01-10', + type: undefined, }, ]); }); diff --git a/src/libs/model-runtime/utils/openaiCompatibleFactory/index.ts b/src/libs/model-runtime/utils/openaiCompatibleFactory/index.ts index fa63c268b3..842cdaedc4 100644 --- a/src/libs/model-runtime/utils/openaiCompatibleFactory/index.ts +++ b/src/libs/model-runtime/utils/openaiCompatibleFactory/index.ts @@ -1,9 +1,11 @@ import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; +import createDebug from 'debug'; import OpenAI, { ClientOptions } from 'openai'; import { Stream } from 'openai/streaming'; -import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders'; +import { LOBE_DEFAULT_MODEL_LIST } from '@/config/aiModels'; +import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/meta-schema'; import type { ChatModelCard } from '@/types/llm'; import { LobeRuntimeAI } from '../../BaseAI'; @@ -27,7 +29,11 @@ import { AgentRuntimeError } from '../createError'; import { debugResponse, debugStream } from '../debugStream'; import { desensitizeUrl } from '../desensitizeUrl'; import { handleOpenAIError } from '../handleOpenAIError'; -import { convertOpenAIMessages, convertOpenAIResponseInputs } from '../openaiHelpers'; +import { + convertImageUrlToFile, + convertOpenAIMessages, + convertOpenAIResponseInputs, +} from '../openaiHelpers'; import { StreamingResponse } from '../response'; import { OpenAIResponsesStream, OpenAIStream, OpenAIStreamOptions } from '../streams'; @@ -168,7 +174,6 @@ export const createOpenAICompatibleRuntime = = an debug, constructorOptions, chatCompletion, - createImage, models, customClient, responses, @@ -311,58 +316,155 @@ export const createOpenAICompatibleRuntime = = an } async createImage(payload: CreateImagePayload) { - return createImage!({ - ...payload, - client: this.client, - }); + const { model, params } = payload; + const log = createDebug(`lobe-image:model-runtime`); + + log('Creating image with model: %s and params: %O', model, params); + + const defaultInput = { + n: 1, + ...(model.includes('dall-e') ? { response_format: 'b64_json' } : {}), + }; + + // 映射参数名称，将 imageUrls 映射为 image + const paramsMap = new Map([ + ['imageUrls', 'image'], + ['imageUrl', 'image'], + ]); + const userInput: Record = Object.fromEntries( + Object.entries(params).map(([key, value]) => [ + paramsMap.get(key as RuntimeImageGenParamsValue) ?? key, + value, + ]), + ); + + const isImageEdit = Array.isArray(userInput.image) && userInput.image.length > 0; + // 如果有 imageUrls 参数，将其转换为 File 对象 + if (isImageEdit) { + log('Converting imageUrls to File objects: %O', userInput.image); + try { + // 转换所有图片 URL 为 File 对象 + const imageFiles = await Promise.all( + userInput.image.map((url: string) => convertImageUrlToFile(url)), + ); + + log('Successfully converted %d images to File objects', imageFiles.length); + + // 根据官方文档，如果有多个图片，传递数组；如果只有一个，传递单个 File + userInput.image = imageFiles.length === 1 ? imageFiles[0] : imageFiles; + } catch (error) { + log('Error converting imageUrls to File objects: %O', error); + throw new Error(`Failed to convert image URLs to File objects: ${error}`); + } + } else { + delete userInput.image; + } + + if (userInput.size === 'auto') { + delete userInput.size; + } + + const options = { + model, + ...defaultInput, + ...userInput, + }; + + log('options: %O', options); + + // 判断是否为图片编辑操作 + const img = isImageEdit + ? await this.client.images.edit(options as any) + : await this.client.images.generate(options as any); + + // 检查响应数据的完整性 + if (!img || !img.data || !Array.isArray(img.data) || img.data.length === 0) { + log('Invalid image response: missing data array'); + throw new Error('Invalid image response: missing or empty data array'); + } + + const imageData = img.data[0]; + if (!imageData) { + log('Invalid image response: first data item is null/undefined'); + throw new Error('Invalid image response: first data item is null or undefined'); + } + + if (!imageData.b64_json) { + log('Invalid image response: missing b64_json field'); + throw new Error('Invalid image response: missing b64_json field'); + } + + // 确定图片的 MIME 类型，默认为 PNG + const mimeType = 'image/png'; // OpenAI 图片生成默认返回 PNG 格式 + + // 将 base64 字符串转换为完整的 data URL + const dataUrl = `data:${mimeType};base64,${imageData.b64_json}`; + + log('Successfully converted base64 to data URL, length: %d', dataUrl.length); + + return { + imageUrl: dataUrl, + }; } async models() { - if (typeof models === 'function') return models({ client: this.client }); - - const list = await this.client.models.list(); - - return list.data - .filter((model) => { - return CHAT_MODELS_BLOCK_LIST.every( - (keyword) => !model.id.toLowerCase().includes(keyword), - ); - }) - .map((item) => { - if (models?.transformModel) { - return models.transformModel(item); - } - - const toReleasedAt = () => { - if (!item.created) return; - dayjs.extend(utc); - - // guarantee item.created in Date String format - if ( - typeof (item.created as any) === 'string' || - // or in milliseconds - item.created.toFixed(0).length === 13 - ) { - return dayjs.utc(item.created).format('YYYY-MM-DD'); + let resultModels: ChatModelCard[] = []; + if (typeof models === 'function') { + resultModels = await models({ client: this.client }); + } else { + const list = await this.client.models.list(); + resultModels = list.data + .filter((model) => { + return CHAT_MODELS_BLOCK_LIST.every( + (keyword) => !model.id.toLowerCase().includes(keyword), + ); + }) + .map((item) => { + if (models?.transformModel) { + return models.transformModel(item); } - // by default, the created time is in seconds - return dayjs.utc(item.created * 1000).format('YYYY-MM-DD'); - }; + const toReleasedAt = () => { + if (!item.created) return; + dayjs.extend(utc); - // TODO: should refactor after remove v1 user/modelList code - const knownModel = LOBE_DEFAULT_MODEL_LIST.find((model) => model.id === item.id); + // guarantee item.created in Date String format + if ( + typeof (item.created as any) === 'string' || + // or in milliseconds + item.created.toFixed(0).length === 13 + ) { + return dayjs.utc(item.created).format('YYYY-MM-DD'); + } - if (knownModel) { - const releasedAt = knownModel.releasedAt ?? toReleasedAt(); + // by default, the created time is in seconds + return dayjs.utc(item.created * 1000).format('YYYY-MM-DD'); + }; - return { ...knownModel, releasedAt }; - } + // TODO: should refactor after remove v1 user/modelList code + const knownModel = LOBE_DEFAULT_MODEL_LIST.find((model) => model.id === item.id); - return { id: item.id, releasedAt: toReleasedAt() }; - }) + if (knownModel) { + const releasedAt = knownModel.releasedAt ?? toReleasedAt(); - .filter(Boolean) as ChatModelCard[]; + return { ...knownModel, releasedAt }; + } + + return { + id: item.id, + releasedAt: toReleasedAt(), + }; + }) + + .filter(Boolean) as ChatModelCard[]; + } + + return resultModels.map((model) => { + return { + ...model, + type: model.type || LOBE_DEFAULT_MODEL_LIST.find((m) => m.id === model.id)?.type, + }; + }) as ChatModelCard[]; } async embeddings( diff --git a/src/libs/model-runtime/utils/openaiHelpers.test.ts b/src/libs/model-runtime/utils/openaiHelpers.test.ts index 85de6248f3..1ee4379e9c 100644 --- a/src/libs/model-runtime/utils/openaiHelpers.test.ts +++ b/src/libs/model-runtime/utils/openaiHelpers.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { imageUrlToBase64 } from '@/utils/imageToBase64'; import { + convertImageUrlToFile, convertMessageContent, convertOpenAIMessages, convertOpenAIResponseInputs, @@ -288,3 +289,153 @@ describe('convertOpenAIResponseInputs', () => { ]); }); }); + +describe('convertImageUrlToFile', () => { + beforeEach(() => { + vi.resetAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('Data URL handling', () => { + it('should convert PNG data URL to File object correctly', async () => { + const base64Data = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='; + const dataUrl = `data:image/png;base64,${base64Data}`; + + const result = await convertImageUrlToFile(dataUrl); + + expect(result).toBeDefined(); + expect(result).toHaveProperty('name', 'image.png'); + expect(result).toHaveProperty('type', 'image/png'); + expect(result).toHaveProperty('size'); + expect(result.size).toBeGreaterThan(0); + }); + + it('should convert JPEG data URL to File object correctly', async () => { + const base64Data = + '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA9BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=='; + const dataUrl = `data:image/jpeg;base64,${base64Data}`; + + const result = await convertImageUrlToFile(dataUrl); + + expect(result).toBeDefined(); + expect(result).toHaveProperty('name', 'image.jpeg'); + expect(result).toHaveProperty('type', 'image/jpeg'); + expect(result).toHaveProperty('size'); + expect(result.size).toBeGreaterThan(0); + }); + + it('should convert WebP data URL to File object correctly', async () => { + const base64Data = 'UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAAAAAJaQAA6g='; + const dataUrl = `data:image/webp;base64,${base64Data}`; + + const result = await convertImageUrlToFile(dataUrl); + + expect(result).toBeDefined(); + expect(result).toHaveProperty('name', 'image.webp'); + expect(result).toHaveProperty('type', 'image/webp'); + expect(result).toHaveProperty('size'); + expect(result.size).toBeGreaterThan(0); + }); + }); + + describe('HTTP URL handling', () => { + const mockFetch = vi.fn(); + + beforeEach(() => { + // Mock global fetch using vi.stubGlobal for better isolation + vi.stubGlobal('fetch', mockFetch); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.clearAllMocks(); + }); + + it('should convert HTTP URL to File object correctly', async () => { + const mockArrayBuffer = new ArrayBuffer(8); + const mockHeaders = new Headers(); + mockHeaders.set('content-type', 'image/jpeg'); + + mockFetch.mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(mockArrayBuffer), + headers: mockHeaders, + } satisfies Partial); + + const result = await convertImageUrlToFile('https://example.com/image.jpg'); + + expect(mockFetch).toHaveBeenCalledWith('https://example.com/image.jpg'); + expect(result).toBeDefined(); + expect(result).toHaveProperty('name', 'image.jpeg'); + expect(result).toHaveProperty('type', 'image/jpeg'); + expect(result).toHaveProperty('size'); + expect(result.size).toEqual(8); + }); + + it('should handle different content types from HTTP response headers', async () => { + const testCases = [ + { contentType: 'image/jpeg', expectedExtension: 'jpeg' }, + { contentType: 'image/png', expectedExtension: 'png' }, + { contentType: 'image/webp', expectedExtension: 'webp' }, + { contentType: null, expectedExtension: 'png' }, // default fallback + ]; + + for (const testCase of testCases) { + const mockArrayBuffer = new ArrayBuffer(8); + const mockHeaders = new Headers(); + if (testCase.contentType) { + mockHeaders.set('content-type', testCase.contentType); + } + + mockFetch.mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(mockArrayBuffer), + headers: mockHeaders, + } satisfies Partial); + + const result = await convertImageUrlToFile('https://example.com/image.jpg'); + + expect(result).toHaveProperty('name', `image.${testCase.expectedExtension}`); + expect(result).toHaveProperty('type', testCase.contentType || 'image/png'); + + vi.clearAllMocks(); + } + }); + + it('should throw error when HTTP request fails', async () => { + mockFetch.mockResolvedValue({ + ok: false, + statusText: 'Not Found', + } satisfies Partial); + + await expect(convertImageUrlToFile('https://example.com/nonexistent.jpg')).rejects.toThrow( + 'Failed to fetch image from https://example.com/nonexistent.jpg: Not Found', + ); + + expect(mockFetch).toHaveBeenCalledWith('https://example.com/nonexistent.jpg'); + }); + + it('should throw error when network request fails', async () => { + mockFetch.mockRejectedValue(new Error('Network error')); + + await expect(convertImageUrlToFile('https://example.com/image.jpg')).rejects.toThrow( + 'Network error', + ); + + expect(mockFetch).toHaveBeenCalledWith('https://example.com/image.jpg'); + }); + }); + + describe('Edge cases', () => { + it('should handle malformed data URL gracefully', async () => { + const malformedDataUrl = 'data:invalid-format'; + + // 这个测试可能会抛出错误，我们需要适当处理 + await expect(convertImageUrlToFile(malformedDataUrl)).rejects.toThrow(); + }); + }); +}); diff --git a/src/libs/model-runtime/utils/openaiHelpers.ts b/src/libs/model-runtime/utils/openaiHelpers.ts index d54c4f0001..6091cbd56c 100644 --- a/src/libs/model-runtime/utils/openaiHelpers.ts +++ b/src/libs/model-runtime/utils/openaiHelpers.ts @@ -1,4 +1,4 @@ -import OpenAI from 'openai'; +import OpenAI, { toFile } from 'openai'; import { disableStreamModels, systemToUserModels } from '@/const/models'; import { ChatStreamPayload, OpenAIChatMessage } from '@/libs/model-runtime'; @@ -119,3 +119,28 @@ export const pruneReasoningPayload = (payload: ChatStreamPayload) => { top_p: 1, }; }; + +/** + * Convert image URL (data URL or HTTP URL) to File object for OpenAI API + */ +export const convertImageUrlToFile = async (imageUrl: string) => { + let buffer: Buffer; + let mimeType: string; + + if (imageUrl.startsWith('data:')) { + // a base64 image + const [mimeTypePart, base64Data] = imageUrl.split(','); + mimeType = mimeTypePart.split(':')[1].split(';')[0]; + buffer = Buffer.from(base64Data, 'base64'); + } else { + // a http url + const response = await fetch(imageUrl); + if (!response.ok) { + throw new Error(`Failed to fetch image from ${imageUrl}: ${response.statusText}`); + } + buffer = Buffer.from(await response.arrayBuffer()); + mimeType = response.headers.get('content-type') || 'image/png'; + } + + return toFile(buffer, `image.${mimeType.split('/')[1]}`, { type: mimeType }); +}; diff --git a/src/libs/model-runtime/xai/index.ts b/src/libs/model-runtime/xai/index.ts index 2a634a78f5..aed3cfb276 100644 --- a/src/libs/model-runtime/xai/index.ts +++ b/src/libs/model-runtime/xai/index.ts @@ -7,10 +7,7 @@ export interface XAIModelCard { id: string; } -export const GrokReasoningModels = new Set([ - 'grok-3-mini', - 'grok-4', -]); +export const GrokReasoningModels = new Set(['grok-3-mini', 'grok-4']); export const isGrokReasoningModel = (model: string) => Array.from(GrokReasoningModels).some((id) => model.includes(id)); diff --git a/src/store/aiInfra/slices/aiModel/action.ts b/src/store/aiInfra/slices/aiModel/action.ts index 6c74be59be..54d0f19fe1 100644 --- a/src/store/aiInfra/slices/aiModel/action.ts +++ b/src/store/aiInfra/slices/aiModel/action.ts @@ -84,7 +84,7 @@ export const createAiModelSlice: StateCreator< }, enabled: model.enabled || false, source: 'remote', - type: 'chat', + type: model.type || 'chat', })), ); diff --git a/src/store/aiInfra/slices/aiProvider/action.ts b/src/store/aiInfra/slices/aiProvider/action.ts index be1c86bbce..785949ed8b 100644 --- a/src/store/aiInfra/slices/aiProvider/action.ts +++ b/src/store/aiInfra/slices/aiProvider/action.ts @@ -203,6 +203,8 @@ export const createAiProviderSlice: StateCreator< onSuccess: async (data) => { if (!data) return; + const { LOBE_DEFAULT_MODEL_LIST } = await import('@/config/aiModels'); + const getModelListByType = (providerId: string, type: string) => { const models = data.enabledAiModels .filter((model) => model.providerId === providerId && model.type === type) @@ -212,7 +214,9 @@ export const createAiProviderSlice: StateCreator< displayName: model.displayName ?? '', id: model.id, ...(model.type === 'image' && { - parameters: (model as AIImageModelCard).parameters, + parameters: + (model as AIImageModelCard).parameters || + LOBE_DEFAULT_MODEL_LIST.find((m) => m.id === model.id)?.parameters, }), })); @@ -231,7 +235,6 @@ export const createAiProviderSlice: StateCreator< children: getModelListByType(provider.id, 'image'), name: provider.name || provider.id, })); - const { LOBE_DEFAULT_MODEL_LIST } = await import('@/config/aiModels'); set( { diff --git a/src/types/aiModel.ts b/src/types/aiModel.ts index c614026ff8..6b4d383079 100644 --- a/src/types/aiModel.ts +++ b/src/types/aiModel.ts @@ -268,6 +268,7 @@ export interface AiFullModelCard extends AIBaseModelCard { displayName?: string; id: string; maxDimension?: number; + parameters?: ModelParamsSchema; pricing?: ChatModelPricing; type: AiModelType; } diff --git a/src/types/llm.ts b/src/types/llm.ts index 231f69ac75..a796cb6937 100644 --- a/src/types/llm.ts +++ b/src/types/llm.ts @@ -1,6 +1,6 @@ import { ReactNode } from 'react'; -import { ChatModelPricing } from '@/types/aiModel'; +import { AiModelType, ChatModelPricing } from '@/types/aiModel'; import { AiProviderSettings } from '@/types/aiProvider'; export type ModelPriceCurrency = 'CNY' | 'USD'; @@ -53,6 +53,8 @@ export interface ChatModelCard { */ releasedAt?: string; + type?: AiModelType; + /** * whether model supports vision */