references/lobe-chat
1
0
Fork 0
mirror of https://github.com/lobehub/lobe-chat.git synced 2026-06-15 12:10:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

🐛 fix: fix claude 3.7 sonnet thinking with tool use (#6528)

Browse Source 
* fix thinking

* try to fix claude thinking with tools calling

* fix

* update

* add tests

* fix tests

* add tests

* add tests

* improve  anthropic
This commit is contained in:
Arvin Xu
2025-02-26 19:04:09 +08:00
committed by GitHub
parent 3659bbc2cb
commit a76d2bff77
18 changed files with 1033 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/(backend)/middleware/auth/index.ts
+6
View File
@@ -23,6 +23,12 @@ export type RequestHandler = (
export const checkAuth =
(handler: RequestHandler) => async (req: Request, options: RequestOptions) => {
// we have a special header to debug the api endpoint in development mode
const isDebugApi = req.headers.get('lobe-auth-dev-backend-api') === '1';
if (process.env.NODE_ENV === 'development' && isDebugApi) {
return handler(req, { ...options, jwtPayload: { userId: 'DEV_USER' } });
}
let jwtPayload: JWTPayload;
try {
src/const/message.ts
+3
View File
@@ -7,3 +7,6 @@ export const MESSAGE_THREAD_DIVIDER_ID = '__THREAD_DIVIDER__';
export const MESSAGE_WELCOME_GUIDE_ID = 'welcome';
export const THREAD_DRAFT_ID = '__THREAD_DRAFT_ID__';
export const MESSAGE_FLAGGED_THINKING='FLAGGED_THINKING'
src/libs/agent-runtime/google/index.test.ts
+8
View File
@@ -449,6 +449,14 @@ describe('LobeGoogleAI', () => {
});
expect(result).toEqual({ text: 'Hello' });
});
it('should handle thinking type messages', async () => {
const result = await instance['convertContentToGooglePart']({
type: 'thinking',
thinking: 'Hello',
signature: 'abc',
});
expect(result).toEqual(undefined);
});
it('should handle base64 type images', async () => {
const base64Image =
src/libs/agent-runtime/google/index.ts
+18 -5
View File
@@ -208,11 +208,18 @@ export class LobeGoogleAI implements LobeRuntimeAI {
system: system_message?.content,
};
}
private convertContentToGooglePart = async (content: UserMessageContentPart): Promise<Part> => {
private convertContentToGooglePart = async (
content: UserMessageContentPart,
): Promise<Part | undefined> => {
switch (content.type) {
default: {
return undefined;
}
case 'text': {
return { text: content.text };
}
case 'image_url': {
const { mimeType, base64, type } = parseDataUri(content.image_url.url);
@@ -261,11 +268,17 @@ export class LobeGoogleAI implements LobeRuntimeAI {
};
}
const getParts = async () => {
if (typeof content === 'string') return [{ text: content }];
const parts = await Promise.all(
content.map(async (c) => await this.convertContentToGooglePart(c)),
);
return parts.filter(Boolean) as Part[];
};
return {
parts:
typeof content === 'string'
? [{ text: content }]
: await Promise.all(content.map(async (c) => await this.convertContentToGooglePart(c))),
parts: await getParts(),
role: message.role === 'assistant' ? 'model' : 'user',
};
};
src/libs/agent-runtime/types/chat.ts
+9 -1
View File
@@ -2,6 +2,11 @@ import { MessageToolCall } from '@/types/message';
export type LLMRoleType = 'user' | 'system' | 'assistant' | 'function' | 'tool';
interface UserMessageContentPartThinking {
signature: string;
thinking: string;
type: 'thinking';
}
interface UserMessageContentPartText {
text: string;
type: 'text';
@@ -15,7 +20,10 @@ interface UserMessageContentPartImage {
type: 'image_url';
}
export type UserMessageContentPart = UserMessageContentPartText | UserMessageContentPartImage;
export type UserMessageContentPart =
| UserMessageContentPartText
| UserMessageContentPartImage
| UserMessageContentPartThinking;
export interface OpenAIChatMessage {
/**
src/libs/agent-runtime/utils/anthropicHelpers.test.ts
+113
View File
@@ -383,6 +383,119 @@ describe('anthropicHelpers', () => {
{ content: '继续', role: 'user' },
]);
});
it('should correctly handle thinking content part', async () => {
const messages: OpenAIChatMessage[] = [
{
content: '告诉我杭州和北京的天气，先回答我好的',
role: 'user',
},
{
content: [
{ thinking: '经过一番思考', type: 'thinking', signature: '123' },
{
type: 'text',
text: '好的,我会为您查询杭州和北京的天气信息。我现在就开始查询这两个城市的当前天气情况。',
},
],
role: 'assistant',
tool_calls: [
{
function: {
arguments: '{"city": "\\u676d\\u5dde"}',
name: 'realtime-weather____fetchCurrentWeather',
},
id: 'toolu_018PNQkH8ChbjoJz4QBiFVod',
type: 'function',
},
{
function: {
arguments: '{"city": "\\u5317\\u4eac"}',
name: 'realtime-weather____fetchCurrentWeather',
},
id: 'toolu_018VQTQ6fwAEC3eppuEfMxPp',
type: 'function',
},
],
},
{
content:
'[{"city":"杭州市","adcode":"330100","province":"浙江","reporttime":"2024-06-24 17:02:14","casts":[{"date":"2024-06-24","week":"1","dayweather":"小雨","nightweather":"中雨","daytemp":"26","nighttemp":"20","daywind":"西","nightwind":"西","daypower":"1-3","nightpower":"1-3","daytemp_float":"26.0","nighttemp_float":"20.0"},{"date":"2024-06-25","week":"2","dayweather":"大雨","nightweather":"中雨","daytemp":"23","nighttemp":"19","daywind":"东","nightwind":"东","daypower":"1-3","nightpower":"1-3","daytemp_float":"23.0","nighttemp_float":"19.0"},{"date":"2024-06-26","week":"3","dayweather":"中雨","nightweather":"中雨","daytemp":"24","nighttemp":"21","daywind":"东南","nightwind":"东南","daypower":"1-3","nightpower":"1-3","daytemp_float":"24.0","nighttemp_float":"21.0"},{"date":"2024-06-27","week":"4","dayweather":"中雨-大雨","nightweather":"中雨","daytemp":"24","nighttemp":"22","daywind":"南","nightwind":"南","daypower":"1-3","nightpower":"1-3","daytemp_float":"24.0","nighttemp_float":"22.0"}]}]',
name: 'realtime-weather____fetchCurrentWeather',
role: 'tool',
tool_call_id: 'toolu_018PNQkH8ChbjoJz4QBiFVod',
},
{
content:
'[{"city":"北京市","adcode":"110000","province":"北京","reporttime":"2024-06-24 17:03:11","casts":[{"date":"2024-06-24","week":"1","dayweather":"晴","nightweather":"晴","daytemp":"33","nighttemp":"20","daywind":"北","nightwind":"北","daypower":"1-3","nightpower":"1-3","daytemp_float":"33.0","nighttemp_float":"20.0"},{"date":"2024-06-25","week":"2","dayweather":"晴","nightweather":"晴","daytemp":"35","nighttemp":"21","daywind":"东南","nightwind":"东南","daypower":"1-3","nightpower":"1-3","daytemp_float":"35.0","nighttemp_float":"21.0"},{"date":"2024-06-26","week":"3","dayweather":"晴","nightweather":"晴","daytemp":"35","nighttemp":"23","daywind":"西南","nightwind":"西南","daypower":"1-3","nightpower":"1-3","daytemp_float":"35.0","nighttemp_float":"23.0"},{"date":"2024-06-27","week":"4","dayweather":"多云","nightweather":"多云","daytemp":"35","nighttemp":"23","daywind":"西南","nightwind":"西南","daypower":"1-3","nightpower":"1-3","daytemp_float":"35.0","nighttemp_float":"23.0"}]}]',
name: 'realtime-weather____fetchCurrentWeather',
role: 'tool',
tool_call_id: 'toolu_018VQTQ6fwAEC3eppuEfMxPp',
},
{
content: '继续',
role: 'user',
},
];
const contents = await buildAnthropicMessages(messages);
expect(contents).toEqual([
{ content: '告诉我杭州和北京的天气，先回答我好的', role: 'user' },
{
content: [
{
signature: '123',
thinking: '经过一番思考',
type: 'thinking',
},
{
text: '好的,我会为您查询杭州和北京的天气信息。我现在就开始查询这两个城市的当前天气情况。',
type: 'text',
},
{
id: 'toolu_018PNQkH8ChbjoJz4QBiFVod',
input: { city: '杭州' },
name: 'realtime-weather____fetchCurrentWeather',
type: 'tool_use',
},
{
id: 'toolu_018VQTQ6fwAEC3eppuEfMxPp',
input: { city: '北京' },
name: 'realtime-weather____fetchCurrentWeather',
type: 'tool_use',
},
],
role: 'assistant',
},
{
content: [
{
content: [
{
text: '[{"city":"杭州市","adcode":"330100","province":"浙江","reporttime":"2024-06-24 17:02:14","casts":[{"date":"2024-06-24","week":"1","dayweather":"小雨","nightweather":"中雨","daytemp":"26","nighttemp":"20","daywind":"西","nightwind":"西","daypower":"1-3","nightpower":"1-3","daytemp_float":"26.0","nighttemp_float":"20.0"},{"date":"2024-06-25","week":"2","dayweather":"大雨","nightweather":"中雨","daytemp":"23","nighttemp":"19","daywind":"东","nightwind":"东","daypower":"1-3","nightpower":"1-3","daytemp_float":"23.0","nighttemp_float":"19.0"},{"date":"2024-06-26","week":"3","dayweather":"中雨","nightweather":"中雨","daytemp":"24","nighttemp":"21","daywind":"东南","nightwind":"东南","daypower":"1-3","nightpower":"1-3","daytemp_float":"24.0","nighttemp_float":"21.0"},{"date":"2024-06-27","week":"4","dayweather":"中雨-大雨","nightweather":"中雨","daytemp":"24","nighttemp":"22","daywind":"南","nightwind":"南","daypower":"1-3","nightpower":"1-3","daytemp_float":"24.0","nighttemp_float":"22.0"}]}]',
type: 'text',
},
],
tool_use_id: 'toolu_018PNQkH8ChbjoJz4QBiFVod',
type: 'tool_result',
},
{
content: [
{
text: '[{"city":"北京市","adcode":"110000","province":"北京","reporttime":"2024-06-24 17:03:11","casts":[{"date":"2024-06-24","week":"1","dayweather":"晴","nightweather":"晴","daytemp":"33","nighttemp":"20","daywind":"北","nightwind":"北","daypower":"1-3","nightpower":"1-3","daytemp_float":"33.0","nighttemp_float":"20.0"},{"date":"2024-06-25","week":"2","dayweather":"晴","nightweather":"晴","daytemp":"35","nighttemp":"21","daywind":"东南","nightwind":"东南","daypower":"1-3","nightpower":"1-3","daytemp_float":"35.0","nighttemp_float":"21.0"},{"date":"2024-06-26","week":"3","dayweather":"晴","nightweather":"晴","daytemp":"35","nighttemp":"23","daywind":"西南","nightwind":"西南","daypower":"1-3","nightpower":"1-3","daytemp_float":"35.0","nighttemp_float":"23.0"},{"date":"2024-06-27","week":"4","dayweather":"多云","nightweather":"多云","daytemp":"35","nighttemp":"23","daywind":"西南","nightwind":"西南","daypower":"1-3","nightpower":"1-3","daytemp_float":"35.0","nighttemp_float":"23.0"}]}]',
type: 'text',
},
],
tool_use_id: 'toolu_018VQTQ6fwAEC3eppuEfMxPp',
type: 'tool_result',
},
],
role: 'user',
},
{ content: '继续', role: 'user' },
]);
});
});
describe('buildAnthropicTools', () => {
src/libs/agent-runtime/utils/anthropicHelpers.ts
+7 -4
View File
@@ -10,6 +10,7 @@ export const buildAnthropicBlock = async (
content: UserMessageContentPart,
): Promise<Anthropic.ContentBlock | Anthropic.ImageBlockParam> => {
switch (content.type) {
case 'thinking':
case 'text': {
// just pass-through the content
return content as any;
@@ -83,13 +84,15 @@ export const buildAnthropicMessage = async (
// if there is tool_calls , we need to covert the tool_calls to tool_use content block
// refs: https://docs.anthropic.com/claude/docs/tool-use#tool-use-and-tool-result-content-blocks
if (message.tool_calls) {
const messageContent =
typeof content === 'string'
? [{ text: message.content, type: 'text' }]
: await Promise.all(content.map(async (c) => await buildAnthropicBlock(c)));
return {
content: [
// avoid empty text content block
!!message.content && {
text: message.content as string,
type: 'text',
},
...messageContent,
...(message.tool_calls.map((tool) => ({
id: tool.id,
input: JSON.parse(tool.function.arguments),
src/libs/agent-runtime/utils/streams/anthropic.test.ts
+361 -116
View File
@@ -384,132 +384,292 @@ describe('AnthropicStream', () => {
expect(onToolCallMock).toHaveBeenCalledTimes(6);
});
it('should handle thinking ', async () => {
const streams = [
{
type: 'message_start',
message: {
id: 'msg_01MNsLe7n1uVLtu6W8rCFujD',
type: 'message',
role: 'assistant',
model: 'claude-3-7-sonnet-20250219',
content: [],
stop_reason: null,
stop_sequence: null,
usage: {
input_tokens: 46,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
output_tokens: 11,
describe('thinking', () => {
it('should handle normal thinking ', async () => {
const streams = [
{
type: 'message_start',
message: {
id: 'msg_01MNsLe7n1uVLtu6W8rCFujD',
type: 'message',
role: 'assistant',
model: 'claude-3-7-sonnet-20250219',
content: [],
stop_reason: null,
stop_sequence: null,
usage: {
input_tokens: 46,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
output_tokens: 11,
},
},
},
},
{
type: 'content_block_start',
index: 0,
content_block: { type: 'thinking', thinking: '', signature: '' },
},
{
type: 'content_block_delta',
index: 0,
delta: { type: 'thinking_delta', thinking: '我需要比较两个数字的' },
},
{
type: 'content_block_delta',
index: 0,
delta: { type: 'thinking_delta', thinking: '大小：9.8和9' },
},
{
type: 'content_block_delta',
index: 0,
delta: { type: 'thinking_delta', thinking: '11\n\n所以9.8比9.11大。' },
},
{
type: 'content_block_delta',
index: 0,
delta: {
type: 'signature_delta',
signature:
'EuYBCkQYAiJAHnHRJG4nPBrdTlo6CmXoyE8WYoQeoPiLnXaeuaM8ExdiIEkVvxK1DYXOz5sCubs2s/G1NsST8A003Zb8XmuhYBIMwDGMZSZ3+gxOEBpVGgzdpOlDNBTxke31SngiMKUk6WcSiA11OSVBuInNukoAhnRd5jPAEg7e5mIoz/qJwnQHV8I+heKUreP77eJdFipQaM3FHn+avEHuLa/Z/fu0O9BftDi+caB1UWDwJakNeWX1yYTvK+N1v4gRpKbj4AhctfYHMjq8qX9XTnXme5AGzCYC6HgYw2/RfalWzwNxI6k=',
{
type: 'content_block_start',
index: 0,
content_block: { type: 'thinking', thinking: '', signature: '' },
},
},
{ type: 'content_block_stop', index: 0 },
{ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } },
{
type: 'content_block_delta',
index: 1,
delta: { type: 'text_delta', text: '9.8比9.11大。' },
},
{ type: 'content_block_stop', index: 1 },
{
type: 'message_delta',
delta: { stop_reason: 'end_turn', stop_sequence: null },
usage: { output_tokens: 354 },
},
{ type: 'message_stop' },
];
{
type: 'content_block_delta',
index: 0,
delta: { type: 'thinking_delta', thinking: '我需要比较两个数字的' },
},
{
type: 'content_block_delta',
index: 0,
delta: { type: 'thinking_delta', thinking: '大小：9.8和9' },
},
{
type: 'content_block_delta',
index: 0,
delta: { type: 'thinking_delta', thinking: '11\n\n所以9.8比9.11大。' },
},
{
type: 'content_block_delta',
index: 0,
delta: {
type: 'signature_delta',
signature:
'EuYBCkQYAiJAHnHRJG4nPBrdTlo6CmXoyE8WYoQeoPiLnXaeuaM8ExdiIEkVvxK1DYXOz5sCubs2s/G1NsST8A003Zb8XmuhYBIMwDGMZSZ3+gxOEBpVGgzdpOlDNBTxke31SngiMKUk6WcSiA11OSVBuInNukoAhnRd5jPAEg7e5mIoz/qJwnQHV8I+heKUreP77eJdFipQaM3FHn+avEHuLa/Z/fu0O9BftDi+caB1UWDwJakNeWX1yYTvK+N1v4gRpKbj4AhctfYHMjq8qX9XTnXme5AGzCYC6HgYw2/RfalWzwNxI6k=',
},
},
{ type: 'content_block_stop', index: 0 },
{ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } },
{
type: 'content_block_delta',
index: 1,
delta: { type: 'text_delta', text: '9.8比9.11大。' },
},
{ type: 'content_block_stop', index: 1 },
{
type: 'message_delta',
delta: { stop_reason: 'end_turn', stop_sequence: null },
usage: { output_tokens: 354 },
},
{ type: 'message_stop' },
];
const mockReadableStream = new ReadableStream({
start(controller) {
streams.forEach((chunk) => {
controller.enqueue(chunk);
});
controller.close();
},
const mockReadableStream = new ReadableStream({
start(controller) {
streams.forEach((chunk) => {
controller.enqueue(chunk);
});
controller.close();
},
});
const protocolStream = AnthropicStream(mockReadableStream);
const decoder = new TextDecoder();
const chunks = [];
// @ts-ignore
for await (const chunk of protocolStream) {
chunks.push(decoder.decode(chunk, { stream: true }));
}
expect(chunks).toEqual(
[
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
'data: {"id":"msg_01MNsLe7n1uVLtu6W8rCFujD","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":46,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":11}}\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: ""\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "我需要比较两个数字的"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "大小：9.8和9"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "11\\n\\n所以9.8比9.11大。"\n',
// Tool calls
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning_signature',
`data: "EuYBCkQYAiJAHnHRJG4nPBrdTlo6CmXoyE8WYoQeoPiLnXaeuaM8ExdiIEkVvxK1DYXOz5sCubs2s/G1NsST8A003Zb8XmuhYBIMwDGMZSZ3+gxOEBpVGgzdpOlDNBTxke31SngiMKUk6WcSiA11OSVBuInNukoAhnRd5jPAEg7e5mIoz/qJwnQHV8I+heKUreP77eJdFipQaM3FHn+avEHuLa/Z/fu0O9BftDi+caB1UWDwJakNeWX1yYTvK+N1v4gRpKbj4AhctfYHMjq8qX9XTnXme5AGzCYC6HgYw2/RfalWzwNxI6k="\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
`data: {"type":"content_block_stop","index":0}\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
`data: ""\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: text',
`data: "9.8比9.11大。"\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
`data: {"type":"content_block_stop","index":1}\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: stop',
'data: "end_turn"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: stop',
'data: "message_stop"\n',
].map((item) => `${item}\n`),
);
});
const protocolStream = AnthropicStream(mockReadableStream);
it('should handle flagged thinking ', async () => {
const streams = [
{
type: 'message_start',
message: {
id: 'msg_019q32esPvu3TftzZnL6JPys',
type: 'message',
role: 'assistant',
model: 'claude-3-7-sonnet-20250219',
content: [],
stop_reason: null,
stop_sequence: null,
usage: {
input_tokens: 92,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
output_tokens: 4,
},
},
},
{
type: 'content_block_start',
index: 0,
content_block: {
type: 'redacted_thinking',
data: 'EvYBCoYBGAIiQNzXoJZW+Ocan2YajVtfm4HE2B3NJdxl05x4M+qDZ2XDAv8uysmma7oaIwNsO/gaZDcaYphIPVvSR0da9BiU4fkqQOseUkmKX3f2PDTFQsTVPGJQdiAoojyYWydq912tQiaWOAnV8pEpsw5qzAhjTg7a/VhucOXRjSO6PrBGUJs4IGgSDEOrVeGKw+XJKwI32RoMXxGUrsCpnzifc238IjCiip27oNxaDKqsGVsa3l8CxznwldGK5o7NKoAWxBr6EjmUyWBfHSjCBSG58dLhH6AqHTHs1h7CpyC9q2PiGFKyI6Qpyq27LMf/IJrL4JzY',
},
},
{ type: 'ping' },
{ type: 'content_block_stop', index: 0 },
{
type: 'content_block_start',
index: 1,
content_block: {
type: 'redacted_thinking',
data: 'EqsCCoYBGAIiQFOzsK5wAM+th5SAo3iYCtupF+/ToOYMoKuQowEkQdMYSr+uTiZGV17Ezt1YopNShapyJHraaanqud0SpjNWb1EqQAIs1xKVmShDP/KzTnkeGj3sB1w9fjEcB8I4Q1oYXmAOvEeBRp+/0eszpC5KM4vfBXockGREIX3b9t0aVkKV5LQSDMMox34k4/t6jt5lwBoM8BCR+z8yvwr8RmRAIjCuZUKwzt5cpTSSKsMRF5w/NkH0KeVbDPkHJAHoyKbVThaz2tNP4DGn9Hje/eOhm14qUjEqjkE7ZBa4oXfutU09Ekn6S+Cn5SsYrFLeg+o4/8ewb8YHuspvYbMMN4IwbkqQp19hi2z6QxUWWbLrpMe40Fi2PNKct/dmGmw/SF692L/tyOU=',
},
},
{ type: 'content_block_stop', index: 1 },
{
type: 'content_block_start',
index: 2,
content_block: {
type: 'redacted_thinking',
data: 'EowCCoYBGAIiQK1Px08f5EwkoGrjGov2SWq2eHJVrkwBhL9atUCuZegNB+yK0F2ENixvwLlFjZOeSDhfVZ3von76crqoGaEUOUgqQGnTe9FWXAOXYnreuT4sCpUCVSq6pyewyyYCJkAVHTc8YCgPQsGagW9qNmUJDNdCoFyMEtFzqRuHZk3nc/9KjJgSDK4yegsNIw6czWXdCxoMzzrg26MN6RjFUrRqIjAjjEWG7mPPMolxAVvscgcaETILV4WtO4xOXDxK0L2NLSb+GlR7LQraWOATBMBc0lAqM43SvsI2xLX6GvdtNIr98tAKXpadetuHoDta+uqVn9dRfJG6Nno0e1cdx9VzgrOM2I0l6w==',
},
},
{ type: 'content_block_stop', index: 2 },
{
type: 'content_block_start',
index: 3,
content_block: {
type: 'redacted_thinking',
data: 'EvMGCoYBGAIiQKkoAFWygajHbTRK/q0hrakXULQBWfg0/EAiNRami4uuzOwDVEPBDu74aP47MMQG0zhLspVkvGpOlfNLkkeROYEqQHO9MLpvtKDkob22tAH2ctP7CxIhI+SRZ0flou71sDdaVtcsel2dIas8+soULHfW68glHJ1ormzeUKv9YHtvVxMSDIC31I3S0nvTPOB/GxoMos7jtbwUPmYvx4viIjD01EiiuBny4srom6xEm/c9VCJQaRKuglEehQ3BRxn2Qs28eGNs7EV63kF5DHV7QTIqmQUaw3A/XKIK+2dhPMzE9/n7VSeWvPl7lFLgTCZBW+q49KoLNuIw5tGMR2nXxTrykvQt9zhNDb9TYAsu8nubMASJt9hWwwMpXAPJhUOP5IL+/p6YDuN9Y5TbDkCiR+3Dgs4xh6VeBhD0cusWdC2LefHT92i1dz2mCFhTtPG8nr/jChOGv/KPPO24sJcSMUYu1T07ohiDCe6vjEckBP2aaSH46rcGEydFBaufPKGD2LsiQfrFDRx639AFlwdeSz30cRrjYCiXBu3l/it0LYt8m5Ixsn41P0xFiPDfecZAkGymvrV8JrS0uPnRbpF9n4CNj1YanoplbVgA9yegj962PnRBHwIoT/UMTLnBgxNE1J9LM6JuMbDQRXpYpZ7OaB9FXwxCKjcWgSiGmiPjdWwan8z7cILDes3Kz9sBaqF4s6uj9eJ31fFL9dHKS0jciCrOPMfKUOQSP/HRuAUsyeyUROquh4MIfXLUUPrFCXyyy42wBvrTXkdWOGZF/wMw6YQGC3iNbgldO4K6OBc8+6+AhRsZR51EuBp1iMl5na6KspyVJnCMx52lUYq3SXNTZkiika/z1jO3C1+cvrzQggo9Yf56bzjKBlVjdjqsIqaNOB8BQqU8EidE668/7cMLF3YJP2YwohEO1C7vOV1vliNkyxdCFz6qB9q8vzZ1hIlFz8LHVxZRmmlMMnAq/Q9nWOXmi/6lIXVRIP+4z6dyIWNINTR/D2ZsMjN34cnDgxgbzGuDoicikliSnJG+RB1smJSAmMrNf+U+JZSW2zpU+7zu1dZm5DMKlef+pmbIJMCxVS7v98vAxt/tO+99HlXwhktL4JuOdC2TcvrDm56e2IeGY0KR5TVA2sfCqxEyb+QAAbwDD7TDwq+r62GVBA==',
},
},
{ type: 'content_block_stop', index: 3 },
{ type: 'content_block_start', index: 4, content_block: { type: 'text', text: '' } },
{
type: 'content_block_delta',
index: 4,
delta: {
type: 'text_delta',
text: "I'm not able to respond to special commands or triggers",
},
},
{
type: 'content_block_delta',
index: 4,
delta: {
type: 'text_delta',
text: ' with information, answer questions, or assist with many other ways.',
},
},
{ type: 'content_block_stop', index: 4 },
{
type: 'message_delta',
delta: { stop_reason: 'end_turn', stop_sequence: null },
usage: { output_tokens: 259 },
},
{ type: 'message_stop' },
];
const decoder = new TextDecoder();
const chunks = [];
const mockReadableStream = new ReadableStream({
start(controller) {
streams.forEach((chunk) => {
controller.enqueue(chunk);
});
controller.close();
},
});
// @ts-ignore
for await (const chunk of protocolStream) {
chunks.push(decoder.decode(chunk, { stream: true }));
}
const protocolStream = AnthropicStream(mockReadableStream);
expect(chunks).toEqual(
[
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
'data: {"id":"msg_01MNsLe7n1uVLtu6W8rCFujD","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":46,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":11}}\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: ""\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "我需要比较两个数字的"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "大小：9.8和9"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "11\\n\\n所以9.8比9.11大。"\n',
// Tool calls
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning_signature',
`data: "EuYBCkQYAiJAHnHRJG4nPBrdTlo6CmXoyE8WYoQeoPiLnXaeuaM8ExdiIEkVvxK1DYXOz5sCubs2s/G1NsST8A003Zb8XmuhYBIMwDGMZSZ3+gxOEBpVGgzdpOlDNBTxke31SngiMKUk6WcSiA11OSVBuInNukoAhnRd5jPAEg7e5mIoz/qJwnQHV8I+heKUreP77eJdFipQaM3FHn+avEHuLa/Z/fu0O9BftDi+caB1UWDwJakNeWX1yYTvK+N1v4gRpKbj4AhctfYHMjq8qX9XTnXme5AGzCYC6HgYw2/RfalWzwNxI6k="\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
`data: {"type":"content_block_stop","index":0}\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
`data: ""\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: text',
`data: "9.8比9.11大。"\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
`data: {"type":"content_block_stop","index":1}\n`,
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: stop',
'data: "end_turn"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: stop',
'data: "message_stop"\n',
].map((item) => `${item}\n`),
);
const decoder = new TextDecoder();
const chunks = [];
// @ts-ignore
for await (const chunk of protocolStream) {
chunks.push(decoder.decode(chunk, { stream: true }));
}
expect(chunks).toEqual(
[
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
'data: {"id":"msg_019q32esPvu3TftzZnL6JPys","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":92,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4}}\n',
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: flagged_reasoning_signature',
'data: "EvYBCoYBGAIiQNzXoJZW+Ocan2YajVtfm4HE2B3NJdxl05x4M+qDZ2XDAv8uysmma7oaIwNsO/gaZDcaYphIPVvSR0da9BiU4fkqQOseUkmKX3f2PDTFQsTVPGJQdiAoojyYWydq912tQiaWOAnV8pEpsw5qzAhjTg7a/VhucOXRjSO6PrBGUJs4IGgSDEOrVeGKw+XJKwI32RoMXxGUrsCpnzifc238IjCiip27oNxaDKqsGVsa3l8CxznwldGK5o7NKoAWxBr6EjmUyWBfHSjCBSG58dLhH6AqHTHs1h7CpyC9q2PiGFKyI6Qpyq27LMf/IJrL4JzY"\n',
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
'data: {"type":"ping"}\n',
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
'data: {"type":"content_block_stop","index":0}\n',
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: flagged_reasoning_signature',
'data: "EqsCCoYBGAIiQFOzsK5wAM+th5SAo3iYCtupF+/ToOYMoKuQowEkQdMYSr+uTiZGV17Ezt1YopNShapyJHraaanqud0SpjNWb1EqQAIs1xKVmShDP/KzTnkeGj3sB1w9fjEcB8I4Q1oYXmAOvEeBRp+/0eszpC5KM4vfBXockGREIX3b9t0aVkKV5LQSDMMox34k4/t6jt5lwBoM8BCR+z8yvwr8RmRAIjCuZUKwzt5cpTSSKsMRF5w/NkH0KeVbDPkHJAHoyKbVThaz2tNP4DGn9Hje/eOhm14qUjEqjkE7ZBa4oXfutU09Ekn6S+Cn5SsYrFLeg+o4/8ewb8YHuspvYbMMN4IwbkqQp19hi2z6QxUWWbLrpMe40Fi2PNKct/dmGmw/SF692L/tyOU="\n',
// Tool calls
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
`data: {"type":"content_block_stop","index":1}\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: flagged_reasoning_signature',
`data: "EowCCoYBGAIiQK1Px08f5EwkoGrjGov2SWq2eHJVrkwBhL9atUCuZegNB+yK0F2ENixvwLlFjZOeSDhfVZ3von76crqoGaEUOUgqQGnTe9FWXAOXYnreuT4sCpUCVSq6pyewyyYCJkAVHTc8YCgPQsGagW9qNmUJDNdCoFyMEtFzqRuHZk3nc/9KjJgSDK4yegsNIw6czWXdCxoMzzrg26MN6RjFUrRqIjAjjEWG7mPPMolxAVvscgcaETILV4WtO4xOXDxK0L2NLSb+GlR7LQraWOATBMBc0lAqM43SvsI2xLX6GvdtNIr98tAKXpadetuHoDta+uqVn9dRfJG6Nno0e1cdx9VzgrOM2I0l6w=="\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
`data: {"type":"content_block_stop","index":2}\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: flagged_reasoning_signature',
`data: "EvMGCoYBGAIiQKkoAFWygajHbTRK/q0hrakXULQBWfg0/EAiNRami4uuzOwDVEPBDu74aP47MMQG0zhLspVkvGpOlfNLkkeROYEqQHO9MLpvtKDkob22tAH2ctP7CxIhI+SRZ0flou71sDdaVtcsel2dIas8+soULHfW68glHJ1ormzeUKv9YHtvVxMSDIC31I3S0nvTPOB/GxoMos7jtbwUPmYvx4viIjD01EiiuBny4srom6xEm/c9VCJQaRKuglEehQ3BRxn2Qs28eGNs7EV63kF5DHV7QTIqmQUaw3A/XKIK+2dhPMzE9/n7VSeWvPl7lFLgTCZBW+q49KoLNuIw5tGMR2nXxTrykvQt9zhNDb9TYAsu8nubMASJt9hWwwMpXAPJhUOP5IL+/p6YDuN9Y5TbDkCiR+3Dgs4xh6VeBhD0cusWdC2LefHT92i1dz2mCFhTtPG8nr/jChOGv/KPPO24sJcSMUYu1T07ohiDCe6vjEckBP2aaSH46rcGEydFBaufPKGD2LsiQfrFDRx639AFlwdeSz30cRrjYCiXBu3l/it0LYt8m5Ixsn41P0xFiPDfecZAkGymvrV8JrS0uPnRbpF9n4CNj1YanoplbVgA9yegj962PnRBHwIoT/UMTLnBgxNE1J9LM6JuMbDQRXpYpZ7OaB9FXwxCKjcWgSiGmiPjdWwan8z7cILDes3Kz9sBaqF4s6uj9eJ31fFL9dHKS0jciCrOPMfKUOQSP/HRuAUsyeyUROquh4MIfXLUUPrFCXyyy42wBvrTXkdWOGZF/wMw6YQGC3iNbgldO4K6OBc8+6+AhRsZR51EuBp1iMl5na6KspyVJnCMx52lUYq3SXNTZkiika/z1jO3C1+cvrzQggo9Yf56bzjKBlVjdjqsIqaNOB8BQqU8EidE668/7cMLF3YJP2YwohEO1C7vOV1vliNkyxdCFz6qB9q8vzZ1hIlFz8LHVxZRmmlMMnAq/Q9nWOXmi/6lIXVRIP+4z6dyIWNINTR/D2ZsMjN34cnDgxgbzGuDoicikliSnJG+RB1smJSAmMrNf+U+JZSW2zpU+7zu1dZm5DMKlef+pmbIJMCxVS7v98vAxt/tO+99HlXwhktL4JuOdC2TcvrDm56e2IeGY0KR5TVA2sfCqxEyb+QAAbwDD7TDwq+r62GVBA=="\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
`data: {"type":"content_block_stop","index":3}\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
`data: ""\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: text',
`data: "I'm not able to respond to special commands or triggers"\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: text',
`data: " with information, answer questions, or assist with many other ways."\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: data',
`data: {"type":"content_block_stop","index":4}\n`,
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: stop',
'data: "end_turn"\n',
'id: msg_019q32esPvu3TftzZnL6JPys',
'event: stop',
'data: "message_stop"\n',
].map((item) => `${item}\n`),
);
});
});
it('should handle ReadableStream input', async () => {
const mockReadableStream = new ReadableStream({
start(controller) {
@@ -550,4 +710,89 @@ describe('AnthropicStream', () => {
`data: "message_stop"\n\n`,
]);
});
it('should handle un-normal block type', async () => {
const streams = [
{
type: 'message_start',
message: {
id: 'msg_01MNsLe7n1uVLtu6W8rCFujD',
type: 'message',
role: 'assistant',
model: 'claude-3-7-sonnet-20250219',
content: [],
stop_reason: null,
stop_sequence: null,
usage: {
input_tokens: 46,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
output_tokens: 11,
},
},
},
{
type: 'content_block_start',
index: 0,
content_block: { type: 'thinking', thinking: 'abc', signature: 'dddd' },
},
{
type: 'content_block_start',
index: 0,
content_block: { type: 'thinking', thinking: null },
},
{
type: 'content_block_start',
index: 0,
content_block: { type: 'abc', abc: '' },
},
{
type: 'content_block_delta',
index: 0,
delta: { type: 'abc', abc: '123' },
},
];
const mockReadableStream = new ReadableStream({
start(controller) {
streams.forEach((chunk) => {
controller.enqueue(chunk);
});
controller.close();
},
});
const protocolStream = AnthropicStream(mockReadableStream);
const decoder = new TextDecoder();
const chunks = [];
// @ts-ignore
for await (const chunk of protocolStream) {
chunks.push(decoder.decode(chunk, { stream: true }));
}
expect(chunks).toEqual(
[
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
'data: {"id":"msg_01MNsLe7n1uVLtu6W8rCFujD","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":46,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":11}}\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning',
'data: "abc"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: reasoning_signature',
'data: "dddd"\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
'data: {"type":"thinking","thinking":null}\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
'data: {"type":"content_block_start","index":0,"content_block":{"type":"abc","abc":""}}\n',
'id: msg_01MNsLe7n1uVLtu6W8rCFujD',
'event: data',
'data: {"type":"content_block_delta","index":0,"delta":{"type":"abc","abc":"123"}}\n',
].map((item) => `${item}\n`),
);
});
});
src/libs/agent-runtime/utils/streams/anthropic.ts
+58 -38
View File
@@ -15,7 +15,7 @@ import {
export const transformAnthropicStream = (
chunk: Anthropic.MessageStreamEvent,
context: StreamContext,
): StreamProtocolChunk => {
): StreamProtocolChunk | StreamProtocolChunk[] => {
// maybe need another structure to add support for multiple choices
switch (chunk.type) {
case 'message_start': {
@@ -23,48 +23,68 @@ export const transformAnthropicStream = (
return { data: chunk.message, id: chunk.message.id, type: 'data' };
}
case 'content_block_start': {
if (chunk.content_block.type === 'tool_use') {
const toolChunk = chunk.content_block;
// if toolIndex is not defined, set it to 0
if (typeof context.toolIndex === 'undefined') {
context.toolIndex = 0;
}
// if toolIndex is defined, increment it
else {
context.toolIndex += 1;
switch (chunk.content_block.type) {
case 'redacted_thinking': {
return {
data: chunk.content_block.data,
id: context.id,
type: 'flagged_reasoning_signature',
};
}
const toolCall: StreamToolCallChunkData = {
function: {
arguments: '',
name: toolChunk.name,
},
id: toolChunk.id,
index: context.toolIndex,
type: 'function',
};
case 'text': {
return { data: chunk.content_block.text, id: context.id, type: 'data' };
}
context.tool = { id: toolChunk.id, index: context.toolIndex, name: toolChunk.name };
case 'tool_use': {
const toolChunk = chunk.content_block;
return { data: [toolCall], id: context.id, type: 'tool_calls' };
// if toolIndex is not defined, set it to 0
if (typeof context.toolIndex === 'undefined') {
context.toolIndex = 0;
}
// if toolIndex is defined, increment it
else {
context.toolIndex += 1;
}
const toolCall: StreamToolCallChunkData = {
function: {
arguments: '',
name: toolChunk.name,
},
id: toolChunk.id,
index: context.toolIndex,
type: 'function',
};
context.tool = { id: toolChunk.id, index: context.toolIndex, name: toolChunk.name };
return { data: [toolCall], id: context.id, type: 'tool_calls' };
}
case 'thinking': {
const thinkingChunk = chunk.content_block;
// if there is signature in the thinking block, return both thinking and signature
if (!!thinkingChunk.signature) {
return [
{ data: thinkingChunk.thinking, id: context.id, type: 'reasoning' },
{ data: thinkingChunk.signature, id: context.id, type: 'reasoning_signature' },
];
}
if (typeof thinkingChunk.thinking === 'string')
return { data: thinkingChunk.thinking, id: context.id, type: 'reasoning' };
return { data: thinkingChunk, id: context.id, type: 'data' };
}
default: {
break;
}
}
if (chunk.content_block.type === 'thinking') {
const thinkingChunk = chunk.content_block;
return { data: thinkingChunk.thinking, id: context.id, type: 'reasoning' };
}
if (chunk.content_block.type === 'redacted_thinking') {
return {
data: chunk.content_block.data,
id: context.id,
type: 'reasoning',
};
}
return { data: chunk.content_block.text, id: context.id, type: 'data' };
return { data: chunk, id: context.id, type: 'data' };
}
case 'content_block_delta': {
@@ -93,7 +113,7 @@ export const transformAnthropicStream = (
return {
data: chunk.delta.signature,
id: context.id,
type: 'reasoning_signature' as any,
type: 'reasoning_signature',
};
}
src/libs/agent-runtime/utils/streams/openai.test.ts
+181
View File
@@ -1576,5 +1576,186 @@ describe('OpenAIStream', () => {
].map((i) => `${i}\n`),
);
});
it('should handle claude reasoning in litellm openai mode', async () => {
const data = [
{
id: '1',
created: 1740505568,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [
{
index: 0,
delta: {
thinking_blocks: [
{ type: 'thinking', thinking: '我需要找94的所有质', ignature_delta: null },
],
reasoning_content: '我需要找出394的所有质',
content: '',
role: 'assistant',
},
},
],
thinking_blocks: [
{ type: 'thinking', thinking: '我需要找94的所有质', signature_delta: null },
],
},
{
id: '1',
created: 1740505569,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [
{
index: 0,
delta: {
thinking_blocks: [
{ type: 'thinking', thinking: '因数。\n质因数是', signature_delta: null },
],
reasoning_content: '因数。\n\n质因数是',
content: '',
},
},
],
thinking_blocks: [
{ type: 'thinking', thinking: '因数。\n\n质因数是', signature_delta: null },
],
},
{
id: '1',
created: 1740505569,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [
{
index: 0,
delta: {
thinking_blocks: [
{ type: 'thinking', thinking: '÷ 2 = 197', signature_delta: null },
],
reasoning_content: '÷ 2 = 197',
content: '',
},
},
],
thinking_blocks: [{ type: 'thinking', thinking: '÷ 2 = 197', signature_delta: null }],
},
{
id: '1',
created: 1740505571,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [
{
index: 0,
delta: {
thinking_blocks: [
{ type: 'thinking', thinking: '197。n394 = 2 ', signature_delta: null },
],
reasoning_content: '197。\n394 = 2 ',
content: '',
},
},
],
thinking_blocks: [{ type: 'thinking', thinking: '\n394 = 2 ', signature_delta: null }],
},
{
id: '1',
created: 1740505571,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [
{
index: 0,
delta: {
content: '',
tool_calls: [{ function: { arguments: '{}' }, type: 'function', index: -1 }],
},
},
],
},
{
id: '1',
created: 1740505571,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [{ index: 0, delta: { content: '要找出394的质因数，我需要将' } }],
},
{
id: '1',
created: 1740505571,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [{ index: 0, delta: { content: '394分解为质数的乘积' } }],
},
{
id: '1',
created: 1740505573,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [{ index: 0, delta: { content: '2和197。' } }],
},
{
id: '1',
created: 1740505573,
model: 'claude-3-7-sonnet-latest',
object: 'chat.completion.chunk',
choices: [{ finish_reason: 'stop', index: 0, delta: {} }],
},
];
const mockOpenAIStream = new ReadableStream({
start(controller) {
data.forEach((chunk) => {
controller.enqueue(chunk);
});
controller.close();
},
});
const protocolStream = OpenAIStream(mockOpenAIStream);
const decoder = new TextDecoder();
const chunks = [];
// @ts-ignore
for await (const chunk of protocolStream) {
chunks.push(decoder.decode(chunk, { stream: true }));
}
expect(chunks).toEqual(
[
'id: 1',
'event: reasoning',
`data: "我需要找出394的所有质"\n`,
'id: 1',
'event: reasoning',
`data: "因数。\\n\\n质因数是"\n`,
'id: 1',
'event: reasoning',
`data: "÷ 2 = 197"\n`,
'id: 1',
'event: reasoning',
`data: "197。\\n394 = 2 "\n`,
'id: 1',
'event: text',
`data: ""\n`,
'id: 1',
'event: text',
`data: "要找出394的质因数，我需要将"\n`,
'id: 1',
'event: text',
`data: "394分解为质数的乘积"\n`,
'id: 1',
'event: text',
`data: "2和197。"\n`,
'id: 1',
'event: stop',
`data: "stop"\n`,
].map((i) => `${i}\n`),
);
});
});
});
src/libs/agent-runtime/utils/streams/openai.ts
+36 -26
View File
@@ -46,36 +46,46 @@ export const transformOpenAIStream = (
// tools calling
if (typeof item.delta?.tool_calls === 'object' && item.delta.tool_calls?.length > 0) {
return {
data: item.delta.tool_calls.map((value, index): StreamToolCallChunkData => {
if (streamContext && !streamContext.tool) {
streamContext.tool = { id: value.id!, index: value.index, name: value.function!.name! };
}
const tool_calls = item.delta.tool_calls.filter(
(value) => value.index >= 0 || typeof value.index === 'undefined',
);
return {
function: {
arguments: value.function?.arguments ?? '{}',
name: value.function?.name ?? null,
},
id:
value.id ||
streamContext?.tool?.id ||
generateToolCallId(index, value.function?.name),
if (tool_calls.length > 0) {
return {
data: item.delta.tool_calls.map((value, index): StreamToolCallChunkData => {
if (streamContext && !streamContext.tool) {
streamContext.tool = {
id: value.id!,
index: value.index,
name: value.function!.name!,
};
}
// mistral's tool calling don't have index and function field, it's data like:
// [{"id":"xbhnmTtY7","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\": [\"A photo of a small, fluffy dog with a playful expression and wagging tail.\", \"A watercolor painting of a small, energetic dog with a glossy coat and bright eyes.\", \"A vector illustration of a small, adorable dog with a short snout and perky ears.\", \"A drawing of a small, scruffy dog with a mischievous grin and a wagging tail.\"], \"quality\": \"standard\", \"seeds\": [123456, 654321, 111222, 333444], \"size\": \"1024x1024\", \"style\": \"vivid\"}"}}]
return {
function: {
arguments: value.function?.arguments ?? '{}',
name: value.function?.name ?? null,
},
id:
value.id ||
streamContext?.tool?.id ||
generateToolCallId(index, value.function?.name),
// minimax's tool calling don't have index field, it's data like:
// [{"id":"call_function_4752059746","type":"function","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\": [\"一个流浪的地球，背景是浩瀚"}}]
// mistral's tool calling don't have index and function field, it's data like:
// [{"id":"xbhnmTtY7","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\": [\"A photo of a small, fluffy dog with a playful expression and wagging tail.\", \"A watercolor painting of a small, energetic dog with a glossy coat and bright eyes.\", \"A vector illustration of a small, adorable dog with a short snout and perky ears.\", \"A drawing of a small, scruffy dog with a mischievous grin and a wagging tail.\"], \"quality\": \"standard\", \"seeds\": [123456, 654321, 111222, 333444], \"size\": \"1024x1024\", \"style\": \"vivid\"}"}}]
// so we need to add these default values
index: typeof value.index !== 'undefined' ? value.index : index,
type: value.type || 'function',
};
}),
id: chunk.id,
type: 'tool_calls',
} as StreamProtocolToolCallChunk;
// minimax's tool calling don't have index field, it's data like:
// [{"id":"call_function_4752059746","type":"function","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\": [\"一个流浪的地球，背景是浩瀚"}}]
// so we need to add these default values
index: typeof value.index !== 'undefined' ? value.index : index,
type: value.type || 'function',
};
}),
id: chunk.id,
type: 'tool_calls',
} as StreamProtocolToolCallChunk;
}
}
// 给定结束原因
src/libs/agent-runtime/utils/streams/protocol.ts
+4
View File
@@ -33,6 +33,10 @@ export interface StreamProtocolChunk {
| 'tool_calls'
// Model Thinking
| 'reasoning'
// use for reasoning signature, maybe only anthropic
| 'reasoning_signature'
// flagged reasoning signature
| 'flagged_reasoning_signature'
// Search or Grounding
| 'grounding'
// stop signal
src/services/__tests__/chat.test.ts
+89 -50
View File
@@ -635,7 +635,7 @@ describe('ChatService', () => {
});
});
describe('processMessage', () => {
describe('reorderToolMessages', () => {
it('should reorderToolMessages', () => {
const input: OpenAIChatMessage[] = [
{
@@ -746,7 +746,9 @@ describe('ChatService', () => {
},
]);
});
});
describe('processMessage', () => {
describe('handle with files content in server mode', () => {
it('should includes files', async () => {
// 重新模拟模块，设置 isServerMode 为 true
@@ -833,46 +835,45 @@ describe('ChatService', () => {
},
]);
});
});
it('should include image files in server mode', async () => {
// 重新模拟模块，设置 isServerMode 为 true
vi.doMock('@/const/version', () => ({
isServerMode: true,
isDeprecatedEdition: true,
}));
it('should include image files in server mode', async () => {
// 重新模拟模块，设置 isServerMode 为 true
vi.doMock('@/const/version', () => ({
isServerMode: true,
isDeprecatedEdition: true,
}));
// 需要在修改模拟后重新导入相关模块
const { chatService } = await import('../chat');
const messages = [
{
content: 'Hello',
role: 'user',
imageList: [
{
id: 'file1',
url: 'http://example.com/image.jpg',
alt: 'abc.png',
},
],
}, // Message with files
{ content: 'Hey', role: 'assistant' }, // Regular user message
] as ChatMessage[];
// 需要在修改模拟后重新导入相关模块
const { chatService } = await import('../chat');
const messages = [
{
content: 'Hello',
role: 'user',
imageList: [
{
id: 'file1',
url: 'http://example.com/image.jpg',
alt: 'abc.png',
},
],
}, // Message with files
{ content: 'Hey', role: 'assistant' }, // Regular user message
] as ChatMessage[];
const getChatCompletionSpy = vi.spyOn(chatService, 'getChatCompletion');
await chatService.createAssistantMessage({
messages,
plugins: [],
model: 'gpt-4-vision-preview',
});
const getChatCompletionSpy = vi.spyOn(chatService, 'getChatCompletion');
await chatService.createAssistantMessage({
messages,
plugins: [],
model: 'gpt-4-vision-preview',
});
expect(getChatCompletionSpy).toHaveBeenCalledWith(
{
messages: [
{
content: [
{
text: `Hello
expect(getChatCompletionSpy).toHaveBeenCalledWith(
{
messages: [
{
content: [
{
text: `Hello
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>following part contains context information injected by the system. Please follow these instructions:
@@ -888,24 +889,61 @@ describe('ChatService', () => {
</files_info>
<!-- END SYSTEM CONTEXT -->`,
type: 'text',
},
{
image_url: { detail: 'auto', url: 'http://example.com/image.jpg' },
type: 'image_url',
},
],
role: 'user',
type: 'text',
},
{
image_url: { detail: 'auto', url: 'http://example.com/image.jpg' },
type: 'image_url',
},
],
role: 'user',
},
{
content: 'Hey',
role: 'assistant',
},
],
model: 'gpt-4-vision-preview',
},
undefined,
);
});
});
it('should handle assistant messages with reasoning correctly', () => {
const messages = [
{
role: 'assistant',
content: 'The answer is 42.',
reasoning: {
content: 'I need to calculate the answer to life, universe, and everything.',
signature: 'thinking_process',
},
},
] as ChatMessage[];
const result = chatService['processMessages']({
messages,
model: 'gpt-4',
provider: 'openai',
});
expect(result).toEqual([
{
content: [
{
signature: 'thinking_process',
thinking: 'I need to calculate the answer to life, universe, and everything.',
type: 'thinking',
},
{
content: 'Hey',
role: 'assistant',
text: 'The answer is 42.',
type: 'text',
},
],
model: 'gpt-4-vision-preview',
role: 'assistant',
},
undefined,
);
]);
});
});
});
@@ -917,6 +955,7 @@ describe('ChatService', () => {
vi.mock('../_auth', async (importOriginal) => {
return importOriginal();
});
describe('AgentRuntimeOnClient', () => {
describe('initializeWithClientStore', () => {
describe('should initialize with options correctly', () => {
src/services/chat.ts
+13 -1
View File
@@ -485,8 +485,20 @@ class ChatService {
}
case 'assistant': {
// signature is a signal of anthropic thinking mode
const shouldIncludeThinking = m.reasoning && !!m.reasoning?.signature;
return {
content: m.content,
content: shouldIncludeThinking
? [
{
signature: m.reasoning!.signature,
thinking: m.reasoning!.content,
type: 'thinking',
} as any,
{ text: m.content, type: 'text' },
]
: m.content,
role: m.role,
tool_calls: m.tools?.map(
(tool): MessageToolCall => ({
src/store/chat/slices/aiChat/actions/generateAIChat.ts
+1 -1
View File
@@ -472,7 +472,7 @@ export const generateAIChat: StateCreator<
// update the content after fetch result
await internal_updateMessageContent(messageId, content, {
toolCalls,
reasoning: !!reasoning ? { content: reasoning, duration } : undefined,
reasoning: !!reasoning ? { ...reasoning, duration } : undefined,
search: !!grounding?.citations ? grounding : undefined,
});
},
src/types/message/base.ts
+1
View File
@@ -10,6 +10,7 @@ export interface CitationItem {
export interface ModelReasoning {
content?: string;
duration?: number;
signature?: string;
}
export type MessageRoleType = 'user' | 'system' | 'assistant' | 'tool';
src/utils/fetch/__tests__/fetchSSE.test.ts
+113 -10
View File
@@ -154,16 +154,119 @@ describe('fetchSSE', () => {
});
});
it('should handle reasoning event with smoothing correctly', async () => {
describe('reasoning', () => {
it('should handle reasoning event without smoothing', async () => {
const mockOnMessageHandle = vi.fn();
const mockOnFinish = vi.fn();
(fetchEventSource as any).mockImplementationOnce(
async (url: string, options: FetchEventSourceInit) => {
options.onopen!({ clone: () => ({ ok: true, headers: new Headers() }) } as any);
options.onmessage!({ event: 'reasoning', data: JSON.stringify('Hello') } as any);
await sleep(100);
options.onmessage!({ event: 'reasoning', data: JSON.stringify(' World') } as any);
await sleep(100);
options.onmessage!({ event: 'text', data: JSON.stringify('hi') } as any);
},
);
await fetchSSE('/', {
onMessageHandle: mockOnMessageHandle,
onFinish: mockOnFinish,
});
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'Hello', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: ' World', type: 'reasoning' });
expect(mockOnFinish).toHaveBeenCalledWith('hi', {
observationId: null,
toolCalls: undefined,
reasoning: { content: 'Hello World' },
traceId: null,
type: 'done',
});
});
it('should handle reasoning event with smoothing correctly', async () => {
const mockOnMessageHandle = vi.fn();
const mockOnFinish = vi.fn();
(fetchEventSource as any).mockImplementationOnce(
async (url: string, options: FetchEventSourceInit) => {
options.onopen!({ clone: () => ({ ok: true, headers: new Headers() }) } as any);
options.onmessage!({ event: 'reasoning', data: JSON.stringify('Hello') } as any);
await sleep(100);
options.onmessage!({ event: 'reasoning', data: JSON.stringify(' World') } as any);
await sleep(100);
options.onmessage!({ event: 'text', data: JSON.stringify('hi') } as any);
},
);
await fetchSSE('/', {
onMessageHandle: mockOnMessageHandle,
onFinish: mockOnFinish,
smoothing: true,
});
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'Hell', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'o', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(3, { text: ' Wor', type: 'reasoning' });
// more assertions for each character...
expect(mockOnFinish).toHaveBeenCalledWith('hi', {
observationId: null,
toolCalls: undefined,
reasoning: { content: 'Hello World' },
traceId: null,
type: 'done',
});
});
it('should handle reasoning with signature', async () => {
const mockOnMessageHandle = vi.fn();
const mockOnFinish = vi.fn();
(fetchEventSource as any).mockImplementationOnce(
async (url: string, options: FetchEventSourceInit) => {
options.onopen!({ clone: () => ({ ok: true, headers: new Headers() }) } as any);
options.onmessage!({ event: 'reasoning', data: JSON.stringify('Hello') } as any);
await sleep(100);
options.onmessage!({ event: 'reasoning', data: JSON.stringify(' World') } as any);
options.onmessage!({
event: 'reasoning_signature',
data: JSON.stringify('abcbcd'),
} as any);
await sleep(100);
options.onmessage!({ event: 'text', data: JSON.stringify('hi') } as any);
},
);
await fetchSSE('/', {
onMessageHandle: mockOnMessageHandle,
onFinish: mockOnFinish,
smoothing: true,
});
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'Hell', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'o', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(3, { text: ' Wor', type: 'reasoning' });
// more assertions for each character...
expect(mockOnFinish).toHaveBeenCalledWith('hi', {
observationId: null,
toolCalls: undefined,
reasoning: { content: 'Hello World', signature: 'abcbcd' },
traceId: null,
type: 'done',
});
});
});
it('should handle grounding event', async () => {
const mockOnMessageHandle = vi.fn();
const mockOnFinish = vi.fn();
(fetchEventSource as any).mockImplementationOnce(
async (url: string, options: FetchEventSourceInit) => {
options.onopen!({ clone: () => ({ ok: true, headers: new Headers() }) } as any);
options.onmessage!({ event: 'reasoning', data: JSON.stringify('Hello') } as any);
await sleep(100);
options.onmessage!({ event: 'reasoning', data: JSON.stringify(' World') } as any);
options.onmessage!({ event: 'grounding', data: JSON.stringify('Hello') } as any);
await sleep(100);
options.onmessage!({ event: 'text', data: JSON.stringify('hi') } as any);
},
@@ -172,17 +275,17 @@ describe('fetchSSE', () => {
await fetchSSE('/', {
onMessageHandle: mockOnMessageHandle,
onFinish: mockOnFinish,
smoothing: true,
});
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'Hell', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'o', type: 'reasoning' });
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(3, { text: ' Wor', type: 'reasoning' });
// more assertions for each character...
expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, {
grounding: 'Hello',
type: 'grounding',
});
expect(mockOnFinish).toHaveBeenCalledWith('hi', {
observationId: null,
toolCalls: undefined,
reasoning: 'Hello World',
grounding: 'Hello',
traceId: null,
type: 'done',
});
src/utils/fetch/fetchSSE.ts
+12 -3
View File
@@ -9,6 +9,7 @@ import {
MessageToolCall,
MessageToolCallChunk,
MessageToolCallSchema,
ModelReasoning,
} from '@/types/message';
import { GroundingSearch } from '@/types/search';
@@ -23,7 +24,7 @@ export type OnFinishHandler = (
context: {
grounding?: GroundingSearch;
observationId?: string | null;
reasoning?: string;
reasoning?: ModelReasoning;
toolCalls?: MessageToolCall[];
traceId?: string | null;
type?: SSEFinishType;
@@ -36,7 +37,8 @@ export interface MessageTextChunk {
}
export interface MessageReasoningChunk {
text: string;
signature?: string;
text?: string;
type: 'reasoning';
}
@@ -271,6 +273,8 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
});
let thinking = '';
let thinkingSignature: string | undefined;
const thinkingController = createSmoothMessage({
onTextUpdate: (delta, text) => {
thinking = text;
@@ -365,6 +369,11 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
break;
}
case 'reasoning_signature': {
thinkingSignature = data;
break;
}
case 'reasoning': {
if (textSmoothing) {
thinkingController.pushToQueue(data);
@@ -436,7 +445,7 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
await options?.onFinish?.(output, {
grounding,
observationId,
reasoning: !!thinking ? thinking : undefined,
reasoning: !!thinking ? { content: thinking, signature: thinkingSignature } : undefined,
toolCalls,
traceId,
type: finishedType,