diff --git a/.env.example b/.env.example index 8758810bed..39946ddc0e 100644 --- a/.env.example +++ b/.env.example @@ -223,6 +223,29 @@ OPENAI_API_KEY=sk-xxxxxxxxx # The LobeChat agents market index url # AGENTS_INDEX_URL=https://chat-agents.lobehub.com +# ####################################### +# ######### Cloud Sandbox Service ####### +# ####################################### + +# Sandbox provider for built-in code execution, shell, file operations, and export. +# Supported values: market, onlyboxes +# SANDBOX_PROVIDER=market + +# Required when SANDBOX_PROVIDER=onlyboxes. Base URL of the Onlyboxes console API, without /api/v1. +# ONLYBOXES_BASE_URL=https://onlyboxes.example.com + +# Required when SANDBOX_PROVIDER=onlyboxes. Must match Onlyboxes CONSOLE_JIT_SIGNING_KEY. +# ONLYBOXES_JIT_SIGNING_KEY=onlyboxes-jit-signing-secret + +# Optional JIT token issuer. Defaults to APP_URL. +# ONLYBOXES_JIT_ISSUER=https://lobehub.example.com + +# Optional JIT token TTL in seconds. +# ONLYBOXES_JIT_TTL_SEC=1800 + +# Optional terminal session lease in seconds for the Onlyboxes provider. +# ONLYBOXES_LEASE_TTL_SEC=900 + # ####################################### # ########### Plugin Service ############ # ####################################### diff --git a/Dockerfile b/Dockerfile index 46cb454d63..8916bd9667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -210,6 +210,14 @@ ENV NEXT_PUBLIC_S3_DOMAIN="" \ S3_ENABLE_PATH_STYLE="" \ S3_SET_ACL="" +# Cloud Sandbox +ENV SANDBOX_PROVIDER="" \ + ONLYBOXES_BASE_URL="" \ + ONLYBOXES_JIT_ISSUER="" \ + ONLYBOXES_JIT_SIGNING_KEY="" \ + ONLYBOXES_JIT_TTL_SEC="" \ + ONLYBOXES_LEASE_TTL_SEC="" + # Model Variables ENV \ # AI21 diff --git a/docs/self-hosting/environment-variables.mdx b/docs/self-hosting/environment-variables.mdx index ee936fac1b..e35f6aa264 100644 --- a/docs/self-hosting/environment-variables.mdx +++ b/docs/self-hosting/environment-variables.mdx @@ -19,11 +19,13 @@ LobeHub provides some additional configuration options when deployed, which can - + - + - + + + ## Building a Custom Image with Overridden `NEXT_PUBLIC` Variables diff --git a/docs/self-hosting/environment-variables.zh-CN.mdx b/docs/self-hosting/environment-variables.zh-CN.mdx index d685f2dc18..ba9194f915 100644 --- a/docs/self-hosting/environment-variables.zh-CN.mdx +++ b/docs/self-hosting/environment-variables.zh-CN.mdx @@ -13,13 +13,15 @@ tags: LobeHub 在部署时提供了一些额外的配置项，你可以使用环境变量进行自定义设置。 - + - + - + - + - + + + diff --git a/docs/self-hosting/environment-variables/cloud-sandbox.mdx b/docs/self-hosting/environment-variables/cloud-sandbox.mdx new file mode 100644 index 0000000000..e1899b3a78 --- /dev/null +++ b/docs/self-hosting/environment-variables/cloud-sandbox.mdx @@ -0,0 +1,136 @@ +--- +title: Configuring Cloud Sandbox +description: Configure the built-in Cloud Sandbox provider, including Market and self-hosted Onlyboxes. +tags: + - Cloud Sandbox + - Onlyboxes + - Self-hosting +--- + +# Configuring Cloud Sandbox + +Cloud Sandbox powers the built-in code execution, shell command, file operation, and file export tools. By default, LobeHub uses the Market sandbox. Self-hosted deployments can switch the same tool surface to an Onlyboxes-compatible sandbox provider. + +## Core Environment Variables + +### `SANDBOX_PROVIDER` + +- Type: Optional +- Description: Selects the server-side sandbox provider. +- Default: `market` +- Example: `onlyboxes` + +Supported values: + +- `market`: Use the existing Market sandbox. +- `onlyboxes`: Use an Onlyboxes-compatible self-hosted sandbox console. + +### `MARKET_BASE_URL` + +- Type: Optional +- Description: Base URL of the Market service. Leave it unset when using the official Market; set it only when connecting to a self-hosted or dedicated Market service. +- Default: `https://market.lobehub.com` +- Example: `https://market.example.com` + +### `MARKET_TRUSTED_CLIENT_ID` + +- Type: Optional +- Description: Market Trusted Client ID used by the LobeHub server to call Market capabilities on behalf of the current user. It must be registered by the target Market service. +- Default: - +- Example: `lobechat-com` + +### `MARKET_TRUSTED_CLIENT_SECRET` + +- Type: Optional +- Description: Shared secret for the Market Trusted Client. It must match the target Market service configuration. +- Default: - +- Example: `your-market-trusted-client-secret` + +### `ONLYBOXES_BASE_URL` + +- Type: Required when `SANDBOX_PROVIDER=onlyboxes` +- Description: Base URL of the Onlyboxes console API. Do not include `/api/v1`. +- Default: - +- Example: `https://onlyboxes.example.com` + +### `ONLYBOXES_JIT_SIGNING_KEY` + +- Type: Required when `SANDBOX_PROVIDER=onlyboxes` +- Description: HMAC signing key used to mint Onlyboxes MCP JIT bearer tokens. It must match the Onlyboxes console `CONSOLE_JIT_SIGNING_KEY`. +- Default: - +- Example: `onlyboxes-jit-signing-secret` + +### `ONLYBOXES_JIT_ISSUER` + +- Type: Optional +- Description: Issuer used in Onlyboxes JIT token claims. +- Default: `APP_URL` +- Example: `https://lobehub.example.com` + +### `ONLYBOXES_JIT_TTL_SEC` + +- Type: Optional +- Description: Lifetime of each Onlyboxes JIT token minted by LobeHub. +- Default: `1800` +- Example: `900` + +### `ONLYBOXES_LEASE_TTL_SEC` + +- Type: Optional +- Description: Lease duration for persistent terminal sessions created by the Cloud Sandbox provider. +- Default: `900` +- Example: `3600` + +## Market Configuration + +By default, LobeHub uses the official Market sandbox and does not require extra sandbox configuration: + +```bash +# SANDBOX_PROVIDER=market +``` + +To explicitly use Market, or to connect to a self-hosted or dedicated Market service, configure: + +```bash +SANDBOX_PROVIDER=market +MARKET_BASE_URL=https://market.example.com +``` + +If that Market service requires the LobeHub server to call sandbox, credential, or skill capabilities on behalf of the current user, also configure Trusted Client credentials: + +```bash +MARKET_TRUSTED_CLIENT_ID=lobechat-com +MARKET_TRUSTED_CLIENT_SECRET=your-market-trusted-client-secret +``` + +`MARKET_TRUSTED_CLIENT_ID` must be registered in the Market service's trusted client allowlist, and `MARKET_TRUSTED_CLIENT_SECRET` must match the shared secret configured on the Market service. Without Trusted Client credentials, Market capabilities that require authentication continue to use the existing user authorization flow. + +## Onlyboxes Runtime Requirements + +The configured Onlyboxes worker should expose `terminalExec` and `terminalResource`. LobeHub uses `terminalExec` as the compatibility layer for shell commands, code execution, and file operations, and uses `terminalResource` for file export through a pre-signed upload URL. + +For feature parity with the Market sandbox, the terminal runtime image should include: + +- `python3`, used by file operation wrappers and Python execution +- `node`, used by JavaScript execution +- `npx` with access to `tsx`, used by TypeScript execution +- Standard shell utilities such as `base64`, `find`, and `grep` + +Minimum configuration for using Onlyboxes: + +```bash +SANDBOX_PROVIDER=onlyboxes +ONLYBOXES_BASE_URL=https://onlyboxes.example.com +ONLYBOXES_JIT_SIGNING_KEY=onlyboxes-jit-signing-secret +``` + +Set the same secret on the Onlyboxes console: + +```bash +CONSOLE_JIT_SIGNING_KEY=onlyboxes-jit-signing-secret +``` + + + File export still writes the exported artifact to the configured LobeHub S3 storage. Configure the + S3 environment variables when users need to download files generated inside the sandbox. + diff --git a/docs/self-hosting/environment-variables/cloud-sandbox.zh-CN.mdx b/docs/self-hosting/environment-variables/cloud-sandbox.zh-CN.mdx new file mode 100644 index 0000000000..ff8c831051 --- /dev/null +++ b/docs/self-hosting/environment-variables/cloud-sandbox.zh-CN.mdx @@ -0,0 +1,136 @@ +--- +title: 配置云端沙箱 +description: 配置内置云端沙箱能力，包括 Market 和自托管 Onlyboxes。 +tags: + - 云端沙箱 + - Onlyboxes + - 自托管 +--- + +# 配置云端沙箱 + +云端沙箱用于内置的代码执行、Shell 命令、文件操作和文件导出工具。默认情况下，LobeHub 使用 Market 沙箱；自托管部署可以把同一套工具能力切换到兼容 Onlyboxes 的沙箱提供方。 + +## 核心环境变量 + +### `SANDBOX_PROVIDER` + +- 类型：可选 +- 描述：选择服务端使用的沙箱提供方。 +- 默认值：`market` +- 示例：`onlyboxes` + +支持的取值： + +- `market`：使用现有 Market 沙箱。 +- `onlyboxes`：使用兼容 Onlyboxes 的自托管沙箱 Console。 + +### `MARKET_BASE_URL` + +- 类型：可选 +- 描述：Market 服务的基础 URL。使用官方 Market 时无需配置；仅当你需要连接自托管或专用 Market 服务时设置。 +- 默认值：`https://market.lobehub.com` +- 示例：`https://market.example.com` + +### `MARKET_TRUSTED_CLIENT_ID` + +- 类型：可选 +- 描述：Market Trusted Client 的客户端 ID，用于让 LobeHub 服务端代表当前用户调用 Market 能力。需要由对应 Market 服务登记。 +- 默认值：- +- 示例：`lobechat-com` + +### `MARKET_TRUSTED_CLIENT_SECRET` + +- 类型：可选 +- 描述：Market Trusted Client 的共享密钥，必须与对应 Market 服务端配置一致。 +- 默认值：- +- 示例：`your-market-trusted-client-secret` + +### `ONLYBOXES_BASE_URL` + +- 类型：当 `SANDBOX_PROVIDER=onlyboxes` 时必填 +- 描述：Onlyboxes Console API 的基础 URL，不需要包含 `/api/v1`。 +- 默认值：- +- 示例：`https://onlyboxes.example.com` + +### `ONLYBOXES_JIT_SIGNING_KEY` + +- 类型：当 `SANDBOX_PROVIDER=onlyboxes` 时必填 +- 描述：用于签发 Onlyboxes MCP JIT Bearer Token 的 HMAC 密钥，必须与 Onlyboxes Console 的 `CONSOLE_JIT_SIGNING_KEY` 一致。 +- 默认值：- +- 示例：`onlyboxes-jit-signing-secret` + +### `ONLYBOXES_JIT_ISSUER` + +- 类型：可选 +- 描述：Onlyboxes JIT Token claims 中使用的签发方。 +- 默认值：`APP_URL` +- 示例：`https://lobehub.example.com` + +### `ONLYBOXES_JIT_TTL_SEC` + +- 类型：可选 +- 描述：LobeHub 签发的每个 Onlyboxes JIT Token 的有效期。 +- 默认值：`1800` +- 示例：`900` + +### `ONLYBOXES_LEASE_TTL_SEC` + +- 类型：可选 +- 描述：云端沙箱提供方创建持久终端会话时使用的租约时长。 +- 默认值：`900` +- 示例：`3600` + +## Market 配置 + +默认情况下，LobeHub 使用官方 Market 沙箱，不需要额外配置： + +```bash +# SANDBOX_PROVIDER=market +``` + +如果你需要显式使用 Market，或者连接自托管 / 专用 Market 服务，可以配置： + +```bash +SANDBOX_PROVIDER=market +MARKET_BASE_URL=https://market.example.com +``` + +如果该 Market 服务要求 LobeHub 服务端代表当前用户调用沙箱、凭据或技能能力，还需要配置 Trusted Client： + +```bash +MARKET_TRUSTED_CLIENT_ID=lobechat-com +MARKET_TRUSTED_CLIENT_SECRET=your-market-trusted-client-secret +``` + +`MARKET_TRUSTED_CLIENT_ID` 需要在 Market 服务端的可信客户端白名单中，`MARKET_TRUSTED_CLIENT_SECRET` 需要与 Market 服务端共享密钥一致。未配置 Trusted Client 时，Market 侧需要认证的能力会继续使用现有的用户授权流程。 + +## Onlyboxes 运行时要求 + +配置的 Onlyboxes worker 需要暴露 `terminalExec` 和 `terminalResource`。LobeHub 使用 `terminalExec` 作为 Shell 命令、代码执行和文件操作的兼容层，并使用 `terminalResource` 通过预签名上传 URL 导出文件。 + +为了和 Market 沙箱保持能力对等，终端运行镜像应包含： + +- `python3`，用于文件操作封装和 Python 执行 +- `node`，用于 JavaScript 执行 +- `npx` 以及可用的 `tsx`，用于 TypeScript 执行 +- `base64`、`find`、`grep` 等常见 Shell 工具 + +使用 Onlyboxes 时的最小配置： + +```bash +SANDBOX_PROVIDER=onlyboxes +ONLYBOXES_BASE_URL=https://onlyboxes.example.com +ONLYBOXES_JIT_SIGNING_KEY=onlyboxes-jit-signing-secret +``` + +Onlyboxes Console 侧需要配置同一个密钥： + +```bash +CONSOLE_JIT_SIGNING_KEY=onlyboxes-jit-signing-secret +``` + + + 文件导出仍会把沙箱内产物写入 LobeHub 配置的 S3 存储。如果用户需要下载沙箱生成的文件，请同时配置 S3 + 相关环境变量。 + diff --git a/package.json b/package.json index b9c0805cbf..16af8fcbba 100644 --- a/package.json +++ b/package.json @@ -277,6 +277,7 @@ "@lobechat/python-interpreter": "workspace:*", "@lobechat/shared-tool-ui": "workspace:*", "@lobechat/ssrf-safe-fetch": "workspace:*", + "@lobechat/tool-runtime": "workspace:*", "@lobechat/utils": "workspace:*", "@lobechat/web-crawler": "workspace:*", "@lobehub/analytics": "^1.6.2", diff --git a/packages/builtin-tool-cloud-sandbox/src/ExecutionRuntime/index.ts b/packages/builtin-tool-cloud-sandbox/src/ExecutionRuntime/index.ts index 11ffd5429d..dce15a1578 100644 --- a/packages/builtin-tool-cloud-sandbox/src/ExecutionRuntime/index.ts +++ b/packages/builtin-tool-cloud-sandbox/src/ExecutionRuntime/index.ts @@ -18,7 +18,7 @@ import type { * * Dependency Injection: * - Client: Inject codeInterpreterService (uses tRPC client) - * - Server: Inject ServerSandboxService (uses MarketSDK directly) + * - Server: Inject configured sandbox provider (Market, Onlyboxes, etc.) */ export class CloudSandboxExecutionRuntime extends ComputerRuntime { private sandboxService: ISandboxService; diff --git a/packages/builtin-tool-cloud-sandbox/src/manifest.ts b/packages/builtin-tool-cloud-sandbox/src/manifest.ts index 7116ee4514..b9891fc9aa 100644 --- a/packages/builtin-tool-cloud-sandbox/src/manifest.ts +++ b/packages/builtin-tool-cloud-sandbox/src/manifest.ts @@ -184,6 +184,7 @@ export const CloudSandboxManifest: BuiltinToolManifest = { }, }, { + defaultTimeoutMs: 120_000, description: 'Execute a shell command and return its output. Supports both synchronous and background execution with timeout control.', humanIntervention: 'required', diff --git a/packages/builtin-tool-cloud-sandbox/src/systemRole.ts b/packages/builtin-tool-cloud-sandbox/src/systemRole.ts index 15e44f97e3..294667a5d9 100644 --- a/packages/builtin-tool-cloud-sandbox/src/systemRole.ts +++ b/packages/builtin-tool-cloud-sandbox/src/systemRole.ts @@ -7,7 +7,7 @@ export const systemPrompt = `You have access to a Cloud Sandbox that provides a - Each conversation topic has its own isolated session - Sessions may expire after inactivity; files will be recreated if needed - The sandbox has its own isolated file system starting at the root directory -- Commands will time out after 60 seconds by default +- Commands will time out after 120 seconds by default - **Default shell is /bin/sh** (typically dash or ash), NOT bash. The \`source\` command may not work as expected. If you need bash-specific features or \`source\`, wrap your command with bash: \`bash -c "source ~/.creds/env && your_command"\` **Credential Injection Locations:** diff --git a/packages/builtin-tool-cloud-sandbox/src/types/service.ts b/packages/builtin-tool-cloud-sandbox/src/types/service.ts index 131aeb3867..d68b24c009 100644 --- a/packages/builtin-tool-cloud-sandbox/src/types/service.ts +++ b/packages/builtin-tool-cloud-sandbox/src/types/service.ts @@ -14,7 +14,7 @@ export interface SandboxCallToolResult { * Result of exporting and uploading a file from sandbox */ export interface SandboxExportFileResult { - error?: { message: string }; + error?: { message: string; name?: string }; fileId?: string; filename: string; mimeType?: string; @@ -29,7 +29,7 @@ export interface SandboxExportFileResult { * Context (topicId, userId) is bound at service creation time, not passed per-call. * This allows CloudSandboxExecutionRuntime to work on both client and server: * - Client: Implemented via tRPC client (codeInterpreterService) - * - Server: Implemented via MarketSDK directly (ServerSandboxService) + * - Server: Implemented via the configured sandbox provider */ export interface ISandboxService { /** diff --git a/packages/tool-runtime/src/ComputerRuntime.ts b/packages/tool-runtime/src/ComputerRuntime.ts index 68eac9f2e7..5ba9cf0ede 100644 --- a/packages/tool-runtime/src/ComputerRuntime.ts +++ b/packages/tool-runtime/src/ComputerRuntime.ts @@ -307,6 +307,7 @@ export abstract class ComputerRuntime { } const r = result.result || {}; + const commandSuccess = typeof r.success === 'boolean' ? r.success : result.success; const state: RunCommandState = { commandId: r.commandId || r.shell_id, @@ -316,7 +317,7 @@ export abstract class ComputerRuntime { output: r.output, stderr: r.stderr, stdout: r.stdout, - success: result.success, + success: commandSuccess, }; const content = formatCommandResult({ @@ -325,7 +326,7 @@ export abstract class ComputerRuntime { shellId: r.commandId || r.shell_id, stderr: r.stderr, stdout: r.stdout || r.output, - success: result.success, + success: commandSuccess, }); return { content, state, success: true }; @@ -346,19 +347,21 @@ export abstract class ComputerRuntime { } const r = result.result || {}; + const outputSuccess = typeof r.success === 'boolean' ? r.success : result.success; const state: GetCommandOutputState = { error: r.error, exitCode: r.exitCode ?? r.exit_code, newOutput: r.newOutput || r.output, - success: result.success, + running: r.running ?? false, + success: outputSuccess, }; const content = formatCommandOutput({ error: r.error, exitCode: r.exitCode ?? r.exit_code, output: r.newOutput || r.output, - success: result.success, + success: outputSuccess, }); return { content, state, success: true }; @@ -379,16 +382,19 @@ export abstract class ComputerRuntime { }); } + const killSuccess = + typeof result.result?.success === 'boolean' ? result.result.success : result.success; + const state: KillCommandState = { commandId: args.commandId, error: result.result?.error, - success: result.success, + success: killSuccess, }; const content = formatKillResult({ error: result.result?.error, shellId: args.commandId, - success: result.success, + success: killSuccess, }); return { content, state, success: true }; diff --git a/packages/tool-runtime/src/types.ts b/packages/tool-runtime/src/types.ts index 246bf3216d..a0e875ba80 100644 --- a/packages/tool-runtime/src/types.ts +++ b/packages/tool-runtime/src/types.ts @@ -196,6 +196,7 @@ export interface GetCommandOutputState { error?: string; exitCode?: number; newOutput?: string; + running?: boolean; success: boolean; } diff --git a/src/envs/__tests__/sandbox.test.ts b/src/envs/__tests__/sandbox.test.ts new file mode 100644 index 0000000000..10dde1ad35 --- /dev/null +++ b/src/envs/__tests__/sandbox.test.ts @@ -0,0 +1,52 @@ +// @vitest-environment node +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +describe('getSandboxConfig', () => { + beforeEach(() => { + vi.resetModules(); + delete process.env.SANDBOX_PROVIDER; + delete process.env.ONLYBOXES_BASE_URL; + delete process.env.ONLYBOXES_JIT_ISSUER; + delete process.env.ONLYBOXES_JIT_SIGNING_KEY; + delete process.env.ONLYBOXES_JIT_TTL_SEC; + delete process.env.ONLYBOXES_LEASE_TTL_SEC; + }); + + it('should treat docker empty string defaults as unset optional values', async () => { + process.env.SANDBOX_PROVIDER = ''; + process.env.ONLYBOXES_BASE_URL = ''; + process.env.ONLYBOXES_JIT_ISSUER = ''; + process.env.ONLYBOXES_JIT_SIGNING_KEY = ''; + process.env.ONLYBOXES_JIT_TTL_SEC = ''; + process.env.ONLYBOXES_LEASE_TTL_SEC = ''; + + const { getSandboxConfig } = await import('../sandbox'); + const config = getSandboxConfig(); + + expect(config.SANDBOX_PROVIDER).toBeUndefined(); + expect(config.ONLYBOXES_BASE_URL).toBeUndefined(); + expect(config.ONLYBOXES_JIT_ISSUER).toBeUndefined(); + expect(config.ONLYBOXES_JIT_SIGNING_KEY).toBeUndefined(); + expect(config.ONLYBOXES_JIT_TTL_SEC).toBeUndefined(); + expect(config.ONLYBOXES_LEASE_TTL_SEC).toBeUndefined(); + }); + + it('should parse configured sandbox values', async () => { + process.env.SANDBOX_PROVIDER = 'onlyboxes'; + process.env.ONLYBOXES_BASE_URL = 'https://onlyboxes.example.com'; + process.env.ONLYBOXES_JIT_ISSUER = 'lobehub-test'; + process.env.ONLYBOXES_JIT_SIGNING_KEY = 'jit-signing-key'; + process.env.ONLYBOXES_JIT_TTL_SEC = '900'; + process.env.ONLYBOXES_LEASE_TTL_SEC = '3600'; + + const { getSandboxConfig } = await import('../sandbox'); + const config = getSandboxConfig(); + + expect(config.SANDBOX_PROVIDER).toBe('onlyboxes'); + expect(config.ONLYBOXES_BASE_URL).toBe('https://onlyboxes.example.com'); + expect(config.ONLYBOXES_JIT_ISSUER).toBe('lobehub-test'); + expect(config.ONLYBOXES_JIT_SIGNING_KEY).toBe('jit-signing-key'); + expect(config.ONLYBOXES_JIT_TTL_SEC).toBe(900); + expect(config.ONLYBOXES_LEASE_TTL_SEC).toBe(3600); + }); +}); diff --git a/src/envs/sandbox.ts b/src/envs/sandbox.ts new file mode 100644 index 0000000000..f469d057dd --- /dev/null +++ b/src/envs/sandbox.ts @@ -0,0 +1,33 @@ +import { createEnv } from '@t3-oss/env-core'; +import { z } from 'zod'; + +const emptyStringToUndefined = (value: unknown) => (value === '' ? undefined : value); + +export const getSandboxConfig = () => { + return createEnv({ + runtimeEnv: { + ONLYBOXES_BASE_URL: process.env.ONLYBOXES_BASE_URL, + ONLYBOXES_JIT_ISSUER: process.env.ONLYBOXES_JIT_ISSUER, + ONLYBOXES_JIT_SIGNING_KEY: process.env.ONLYBOXES_JIT_SIGNING_KEY, + ONLYBOXES_JIT_TTL_SEC: process.env.ONLYBOXES_JIT_TTL_SEC, + ONLYBOXES_LEASE_TTL_SEC: process.env.ONLYBOXES_LEASE_TTL_SEC, + SANDBOX_PROVIDER: process.env.SANDBOX_PROVIDER, + }, + server: { + ONLYBOXES_BASE_URL: z.preprocess(emptyStringToUndefined, z.string().url().optional()), + ONLYBOXES_JIT_ISSUER: z.preprocess(emptyStringToUndefined, z.string().optional()), + ONLYBOXES_JIT_SIGNING_KEY: z.preprocess(emptyStringToUndefined, z.string().optional()), + ONLYBOXES_JIT_TTL_SEC: z.preprocess( + emptyStringToUndefined, + z.coerce.number().int().positive().optional(), + ), + ONLYBOXES_LEASE_TTL_SEC: z.preprocess(emptyStringToUndefined, z.coerce.number().optional()), + SANDBOX_PROVIDER: z.preprocess( + emptyStringToUndefined, + z.enum(['market', 'onlyboxes']).optional(), + ), + }, + }); +}; + +export const sandboxEnv = getSandboxConfig(); diff --git a/src/server/modules/S3/index.test.ts b/src/server/modules/S3/index.test.ts index 6251ac24aa..dcb90a80d2 100644 --- a/src/server/modules/S3/index.test.ts +++ b/src/server/modules/S3/index.test.ts @@ -380,6 +380,24 @@ describe('FileS3', () => { }); }); + describe('createPreSignedUpload', () => { + it('should return upload headers required by the signed PUT request', async () => { + const s3 = new FileS3(); + + const result = await s3.createPreSignedUpload('upload-file.txt'); + + expect(PutObjectCommand).toHaveBeenCalledWith({ + ACL: 'public-read', + Bucket: 'test-bucket', + Key: 'upload-file.txt', + }); + expect(result).toEqual({ + headers: { 'x-amz-acl': 'public-read' }, + url: 'https://presigned-url.example.com', + }); + }); + }); + describe('createPreSignedUrlForPreview', () => { it('should create presigned URL for preview with default expiration', async () => { const s3 = new FileS3(); diff --git a/src/server/modules/S3/index.ts b/src/server/modules/S3/index.ts index b55679b38c..3942f69c3b 100644 --- a/src/server/modules/S3/index.ts +++ b/src/server/modules/S3/index.ts @@ -24,6 +24,12 @@ export const listFileSchema = z.array(fileSchema); export type FileType = z.infer; const DEFAULT_S3_REGION = 'us-east-1'; +const PUBLIC_READ_ACL_HEADER = 'public-read'; + +export interface PreSignedUpload { + headers?: Record; + url: string; +} export class S3 { private readonly client: S3Client; @@ -133,13 +139,23 @@ export class S3 { } public async createPreSignedUrl(key: string): Promise { + const upload = await this.createPreSignedUpload(key); + return upload.url; + } + + public async createPreSignedUpload(key: string): Promise { const command = new PutObjectCommand({ - ACL: this.setAcl ? 'public-read' : undefined, + ACL: this.setAcl ? PUBLIC_READ_ACL_HEADER : undefined, Bucket: this.bucket, Key: key, }); - return getSignedUrl(this.client, command, { expiresIn: 3600 }); + const url = await getSignedUrl(this.client, command, { expiresIn: 3600 }); + + return { + headers: this.setAcl ? { 'x-amz-acl': PUBLIC_READ_ACL_HEADER } : undefined, + url, + }; } public async createPreSignedUrlForPreview(key: string, expiresIn?: number): Promise { diff --git a/src/server/routers/tools/market.ts b/src/server/routers/tools/market.ts index 2015dd6791..3cff4ce095 100644 --- a/src/server/routers/tools/market.ts +++ b/src/server/routers/tools/market.ts @@ -1,8 +1,6 @@ import { MARKET_AUTH_REQUIRED_MESSAGE } from '@lobechat/desktop-bridge'; -import { type CodeInterpreterToolName } from '@lobehub/market-sdk'; import { TRPCError } from '@trpc/server'; import debug from 'debug'; -import { sha256 } from 'js-sha256'; import { z } from 'zod'; import { AgentSkillModel } from '@/database/models/agentSkill'; @@ -12,7 +10,6 @@ import { authedProcedure, router } from '@/libs/trpc/lambda'; import { marketUserInfo, serverDatabase, telemetry } from '@/libs/trpc/lambda/middleware'; import { marketSDK, requireMarketAuth } from '@/libs/trpc/lambda/middleware/marketSDK'; import { isTrustedClientEnabled } from '@/libs/trusted-client'; -import { FileS3 } from '@/server/modules/S3'; import { DiscoverService } from '@/server/services/discover'; import { FileService } from '@/server/services/file'; import { MarketService } from '@/server/services/market'; @@ -20,6 +17,7 @@ import { contentBlocksToString, processContentBlocks, } from '@/server/services/mcp/contentProcessor'; +import { createSandboxService } from '@/server/services/sandbox'; import { scheduleToolCallReport } from './_helpers'; import { @@ -30,6 +28,27 @@ import { const log = debug('lobe-server:tools:market'); +const isSandboxAuthError = (error?: { message?: string; name?: string }) => { + const code = error?.name; + const message = error?.message || ''; + + return ( + code === 'invalid_token' || + code === 'token_expired' || + code === 'unauthorized' || + message.toLowerCase().includes('invalid_token') || + message.toLowerCase().includes('token expired') || + message.toLowerCase().includes('unauthorized') + ); +}; + +const throwSandboxAuthError = () => { + throw new TRPCError({ + code: 'UNAUTHORIZED', + message: MARKET_AUTH_REQUIRED_MESSAGE, + }); +}; + // ============================== Common Procedure ============================== const marketToolProcedure = authedProcedure .use(serverDatabase) @@ -197,7 +216,7 @@ const execInSandboxHandler = async ({ const fullUrl = await ctx.fileService.getFullFileUrl(fileInfo.url); if (fullUrl) { skillZipUrls[activatedSkill.name] = fullUrl; - log('Resolved zipUrl for skill %s: %s', activatedSkill.name, fullUrl); + log('Resolved zipUrl for skill %s', activatedSkill.name); } } @@ -211,50 +230,22 @@ const execInSandboxHandler = async ({ } } - const market = ctx.marketService.market; + const sandboxService = createSandboxService({ + fileService: ctx.fileService, + marketService: ctx.marketService, + topicId, + userId, + }); - const response = await market.plugins.runBuildInTool( - toolName as CodeInterpreterToolName, - enhancedParams as any, - { topicId, userId }, - ); + const response = await sandboxService.callTool(toolName, enhancedParams); log('execInSandbox response for %s: %O', toolName, response); - if (!response.success) { - const errorCode = response.error?.code; - const errorMessage = response.error?.message || 'Unknown error'; - - // Check for authentication errors and throw UNAUTHORIZED to trigger market auth flow - if ( - errorCode === 'invalid_token' || - errorCode === 'token_expired' || - errorCode === 'unauthorized' || - errorMessage.toLowerCase().includes('invalid_token') || - errorMessage.toLowerCase().includes('token expired') - ) { - throw new TRPCError({ - code: 'UNAUTHORIZED', - message: MARKET_AUTH_REQUIRED_MESSAGE, - }); - } - - return { - error: { - message: errorMessage, - name: errorCode, - }, - result: null, - sessionExpiredAndRecreated: false, - success: false, - }; + if (!response.success && isSandboxAuthError(response.error)) { + throwSandboxAuthError(); } - return { - result: response.data?.result, - sessionExpiredAndRecreated: response.data?.sessionExpiredAndRecreated || false, - success: true, - }; + return response; } catch (error) { log('execInSandbox error for %s: %O', toolName, error); @@ -659,94 +650,19 @@ export const marketRouter = router({ log('Exporting and uploading file: %s from path: %s in topic: %s', filename, path, topicId); try { - const s3 = new FileS3(); - - // Use date-based sharding for privacy compliance (GDPR, CCPA) - const today = new Date().toISOString().split('T')[0]; - - // Generate a unique key for the exported file - const key = `code-interpreter-exports/${today}/${topicId}/${filename}`; - - // Step 1: Generate pre-signed upload URL - const uploadUrl = await s3.createPreSignedUrl(key); - log('Generated upload URL for key: %s', key); - - // Step 2: Use MarketService from ctx - const market = ctx.marketService.market; - - // Step 3: Call sandbox's exportFile tool with the upload URL - const response = await market.plugins.runBuildInTool( - 'exportFile', - { path, uploadUrl }, - { topicId, userId: ctx.userId }, - ); - - log('Sandbox exportFile response: %O', response); - - if (!response.success) { - const errorCode = response.error?.code; - const errorMessage = response.error?.message || 'Failed to export file from sandbox'; - - // Check for authentication errors and throw UNAUTHORIZED - if ( - errorCode === 'invalid_token' || - errorCode === 'token_expired' || - errorCode === 'unauthorized' || - errorMessage.toLowerCase().includes('invalid_token') || - errorMessage.toLowerCase().includes('token expired') - ) { - throw new TRPCError({ - code: 'UNAUTHORIZED', - message: - 'Market authorization expired. An authorization dialog has been shown to the user. Please wait for the user to complete authorization and then retry the current task.', - }); - } - - return { - error: { message: errorMessage }, - filename, - success: false, - } as ExportAndUploadFileResult; - } - - const result = response.data?.result; - const uploadSuccess = result?.success !== false; - - if (!uploadSuccess) { - return { - error: { message: result?.error || 'Failed to upload file from sandbox' }, - filename, - success: false, - } as ExportAndUploadFileResult; - } - - // Step 4: Get file metadata from S3 to verify upload and get actual size - const metadata = await s3.getFileMetadata(key); - const fileSize = metadata.contentLength; - const mimeType = metadata.contentType || result?.mimeType || 'application/octet-stream'; - - // Step 5: Create persistent file record using FileService - // Generate a simple hash from the key (since we don't have the actual file content) - const fileHash = sha256(key + Date.now().toString()); - - const { fileId, url } = await ctx.fileService.createFileRecord({ - fileHash, - fileType: mimeType, - name: filename, - size: fileSize, - url: key, // Store S3 key + const sandboxService = createSandboxService({ + fileService: ctx.fileService, + marketService: ctx.marketService, + topicId, + userId: ctx.userId, }); + const result = await sandboxService.exportAndUploadFile(path, filename); - log('Created file record: fileId=%s, url=%s', fileId, url); + if (!result.success && isSandboxAuthError(result.error)) { + throwSandboxAuthError(); + } - return { - fileId, - filename, - mimeType, - size: fileSize, - success: true, - url, // This is the permanent /f/:id URL - } as ExportAndUploadFileResult; + return result as ExportAndUploadFileResult; } catch (error) { log('Error in exportAndUploadFile: %O', error); @@ -763,11 +679,7 @@ export const marketRouter = router({ errorMessage.toLowerCase().includes('token expired') || errorMessage.toLowerCase().includes('unauthorized') ) { - throw new TRPCError({ - code: 'UNAUTHORIZED', - message: - 'Market authorization expired. An authorization dialog has been shown to the user. Please wait for the user to complete authorization and then retry the current task.', - }); + throwSandboxAuthError(); } return { diff --git a/src/server/services/file/__tests__/index.test.ts b/src/server/services/file/__tests__/index.test.ts index 8b43771f13..2118a5278c 100644 --- a/src/server/services/file/__tests__/index.test.ts +++ b/src/server/services/file/__tests__/index.test.ts @@ -25,6 +25,7 @@ vi.mock('../impls', () => ({ getFileContent: vi.fn(), getFileByteArray: vi.fn(), getFileMetadata: vi.fn(), + createPreSignedUpload: vi.fn(), createPreSignedUrl: vi.fn(), createPreSignedUrlForPreview: vi.fn(), createCachedPreSignedUrlForPreview: vi.fn(), @@ -207,6 +208,20 @@ describe('FileService', () => { expect(result).toBe(expectedUrl); }); + it('should delegate createPreSignedUpload to implementation', async () => { + const testKey = 'test-key'; + const expectedUpload = { + headers: { 'x-amz-acl': 'public-read' }, + url: 'https://example.com/signed-url', + }; + vi.mocked(service['impl'].createPreSignedUpload).mockResolvedValue(expectedUpload); + + const result = await service.createPreSignedUpload(testKey); + + expect(service['impl'].createPreSignedUpload).toHaveBeenCalledWith(testKey); + expect(result).toBe(expectedUpload); + }); + it('should delegate createPreSignedUrlForPreview to implementation', async () => { const testKey = 'test-key'; const expiresIn = 3600; diff --git a/src/server/services/file/impls/s3.test.ts b/src/server/services/file/impls/s3.test.ts index 598a4d1b9b..6c9b31ddc2 100644 --- a/src/server/services/file/impls/s3.test.ts +++ b/src/server/services/file/impls/s3.test.ts @@ -49,6 +49,10 @@ vi.mock('@/server/modules/S3', () => ({ getFileMetadata: vi.fn().mockResolvedValue({ contentLength: 1024, contentType: 'image/png' }), deleteFile: vi.fn().mockResolvedValue({}), deleteFiles: vi.fn().mockResolvedValue({}), + createPreSignedUpload: vi.fn().mockResolvedValue({ + headers: { 'x-amz-acl': 'public-read' }, + url: 'https://upload.example.com/test.jpg', + }), createPreSignedUrl: vi.fn().mockResolvedValue('https://upload.example.com/test.jpg'), uploadContent: vi.fn().mockResolvedValue({}), uploadMedia: vi.fn().mockResolvedValue({}), @@ -281,6 +285,18 @@ describe('S3StaticFileImpl', () => { }); }); + describe('createPreSignedUpload', () => { + it('should call S3 createPreSignedUpload and return upload headers', async () => { + const result = await fileService.createPreSignedUpload('test.jpg'); + + expect(fileService['s3'].createPreSignedUpload).toHaveBeenCalledWith('test.jpg'); + expect(result).toEqual({ + headers: { 'x-amz-acl': 'public-read' }, + url: 'https://upload.example.com/test.jpg', + }); + }); + }); + describe('getFileMetadata', () => { it('should call S3 getFileMetadata and return metadata', async () => { const result = await fileService.getFileMetadata('test.png'); diff --git a/src/server/services/file/impls/s3.ts b/src/server/services/file/impls/s3.ts index affd6a805c..ac9c0a914e 100644 --- a/src/server/services/file/impls/s3.ts +++ b/src/server/services/file/impls/s3.ts @@ -8,7 +8,7 @@ import { getRedisConfig } from '@/envs/redis'; import { initializeRedis, isRedisEnabled } from '@/libs/redis'; import { FileS3 } from '@/server/modules/S3'; -import { type FileServiceImpl } from './type'; +import type { FileServiceImpl, PreSignedUpload } from './type'; const log = debug('lobe-file:s3'); @@ -64,6 +64,10 @@ export class S3StaticFileImpl implements FileServiceImpl { return this.s3.createPreSignedUrl(key); } + async createPreSignedUpload(key: string): Promise { + return this.s3.createPreSignedUpload(key); + } + async getFileMetadata(key: string): Promise<{ contentLength: number; contentType?: string }> { return this.s3.getFileMetadata(key); } diff --git a/src/server/services/file/impls/type.ts b/src/server/services/file/impls/type.ts index 2ae449eb70..e513969412 100644 --- a/src/server/services/file/impls/type.ts +++ b/src/server/services/file/impls/type.ts @@ -1,3 +1,7 @@ +import type { PreSignedUpload } from '@/server/modules/S3'; + +export type { PreSignedUpload }; + /** * File service implementation interface */ @@ -6,6 +10,12 @@ export interface FileServiceImpl { * Create cached pre-signed preview URL */ createCachedPreSignedUrlForPreview: (url?: string | null, expiresIn?: number) => Promise; + + /** + * Create pre-signed upload descriptor + */ + createPreSignedUpload: (key: string) => Promise; + /** * Create pre-signed upload URL */ diff --git a/src/server/services/file/index.ts b/src/server/services/file/index.ts index b9bc1bec0e..a99d3a3ace 100644 --- a/src/server/services/file/index.ts +++ b/src/server/services/file/index.ts @@ -11,7 +11,7 @@ import { TempFileManager } from '@/server/utils/tempFileManager'; import { isDev } from '@/utils/env'; import { createFileServiceModule } from './impls'; -import { type FileServiceImpl } from './impls/type'; +import type { FileServiceImpl, PreSignedUpload } from './impls/type'; export const getFileProxyUrl = (fileId: string): string => `${appEnv.APP_URL}/f/${fileId}`; @@ -72,6 +72,13 @@ export class FileService { return this.impl.createPreSignedUrl(key); } + /** + * Create pre-signed upload descriptor + */ + public async createPreSignedUpload(key: string): Promise { + return this.impl.createPreSignedUpload(key); + } + /** * Get file metadata from storage * Used to verify actual file size instead of trusting client-provided values diff --git a/src/server/services/heterogeneousAgent/sandboxRunner.ts b/src/server/services/heterogeneousAgent/sandboxRunner.ts index 06ed819bd0..b1dfbfa4aa 100644 --- a/src/server/services/heterogeneousAgent/sandboxRunner.ts +++ b/src/server/services/heterogeneousAgent/sandboxRunner.ts @@ -2,6 +2,7 @@ import debug from 'debug'; import { appEnv } from '@/envs/app'; import type { MarketService } from '@/server/services/market'; +import { createSandboxService } from '@/server/services/sandbox'; const log = debug('lobe-server:hetero-sandbox-runner'); @@ -96,8 +97,8 @@ function buildRepoSetupScript(repos: string[], githubToken?: string): string | n /** * Launches `lh hetero exec` inside the cloud sandbox via `runCommand`. * - * Uses the same MarketService path as ServerSandboxService.callTool — - * `marketService.getSDK().plugins.runBuildInTool('runCommand', params, ctx)`. + * Uses the configured sandbox provider so cloud, third-party, and self-hosted + * sandboxes share the same launch path. * * The sandbox container already has `lh` (the LobeHub CLI) installed. * The operation-scoped JWT is injected as `LOBEHUB_JWT` so the CLI can @@ -192,7 +193,14 @@ export async function spawnHeteroSandbox(params: SandboxRunParams): Promise { + return this.serviceResult; + } +} + +describe('ComputerRuntime command status mapping', () => { + it('uses command result success when command transport succeeds with non-zero exit code', async () => { + const runtime = new TestComputerRuntime({ + result: { + exitCode: 2, + stderr: 'failed', + stdout: 'partial', + success: false, + }, + success: true, + }); + + const result = await runtime.runCommand({ command: 'exit 2' }); + + expect(result).toMatchObject({ + state: { + exitCode: 2, + stderr: 'failed', + stdout: 'partial', + success: false, + }, + success: true, + }); + }); + + it('uses command output result success when background task transport succeeds', async () => { + const runtime = new TestComputerRuntime({ + result: { + output: 'failed', + running: false, + success: false, + }, + success: true, + }); + + const result = await runtime.getCommandOutput({ commandId: 'task-1' }); + + expect(result).toMatchObject({ + state: { + newOutput: 'failed', + running: false, + success: false, + }, + success: true, + }); + }); +}); diff --git a/src/server/services/sandbox/__tests__/factory.test.ts b/src/server/services/sandbox/__tests__/factory.test.ts new file mode 100644 index 0000000000..ad0c0cc1bb --- /dev/null +++ b/src/server/services/sandbox/__tests__/factory.test.ts @@ -0,0 +1,55 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MarketService } from '@/server/services/market'; + +const baseOptions = { + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', +}; + +describe('sandbox service factory', () => { + beforeEach(() => { + vi.resetModules(); + }); + + it('uses the market provider by default', async () => { + vi.doMock('@/envs/sandbox', () => ({ + sandboxEnv: {}, + })); + + const { createSandboxService } = await import('../factory'); + const service = createSandboxService(baseOptions); + + expect(service.kind).toBe('market'); + expect(service.capabilities).toMatchObject({ + backgroundCommands: true, + exportFile: true, + files: true, + persistentSession: true, + shell: true, + skillScripts: true, + }); + }); + + it('uses the onlyboxes provider when configured', async () => { + vi.doMock('@/envs/app', () => ({ + appEnv: { + APP_URL: 'https://lobehub.example.com', + }, + })); + vi.doMock('@/envs/sandbox', () => ({ + sandboxEnv: { + ONLYBOXES_BASE_URL: 'https://onlyboxes.example.com', + ONLYBOXES_JIT_SIGNING_KEY: 'jit-signing-key', + SANDBOX_PROVIDER: 'onlyboxes', + }, + })); + + const { createSandboxService } = await import('../factory'); + const service = createSandboxService(baseOptions); + + expect(service.kind).toBe('onlyboxes'); + expect(service.capabilities.languages).toEqual(['python', 'javascript', 'typescript']); + }); +}); diff --git a/src/server/services/sandbox/__tests__/service.test.ts b/src/server/services/sandbox/__tests__/service.test.ts new file mode 100644 index 0000000000..aefdbb02cd --- /dev/null +++ b/src/server/services/sandbox/__tests__/service.test.ts @@ -0,0 +1,125 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { FileService } from '@/server/services/file'; +import type { MarketService } from '@/server/services/market'; + +import type { SandboxProvider } from '../types'; + +describe('SandboxMiddlewareService', () => { + beforeEach(() => { + vi.resetModules(); + vi.restoreAllMocks(); + }); + + it('uploads provider exports through the shared file record flow', async () => { + const { SandboxMiddlewareService: TestSandboxMiddlewareService } = await import('../service'); + const exportFileToUploadUrl = vi.fn(async () => ({ + result: { mime_type: 'text/plain' }, + success: true, + })); + const provider = { + capabilities: { + backgroundCommands: true, + exportFile: true, + files: true, + languages: ['python'], + persistentSession: true, + shell: true, + skillScripts: true, + }, + callTool: vi.fn(), + exportFileToUploadUrl, + kind: 'onlyboxes', + } satisfies SandboxProvider; + + const fileService = { + createPreSignedUpload: vi.fn(async () => ({ + headers: { 'x-amz-acl': 'public-read' }, + url: 'https://uploads.example.com/put', + })), + createFileRecord: vi.fn(async () => ({ fileId: 'file-1', url: '/f/file-1' })), + getFileMetadata: vi.fn(async () => ({ + contentLength: 42, + contentType: 'text/csv', + })), + } as unknown as FileService; + + const service = new TestSandboxMiddlewareService(provider, { + fileService, + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await service.exportAndUploadFile('/workspace/result.csv', 'result.csv'); + + expect(result).toMatchObject({ + fileId: 'file-1', + filename: 'result.csv', + mimeType: 'text/csv', + size: 42, + success: true, + url: '/f/file-1', + }); + expect(exportFileToUploadUrl).toHaveBeenCalledWith({ + filename: 'result.csv', + path: '/workspace/result.csv', + uploadHeaders: { 'x-amz-acl': 'public-read' }, + uploadUrl: 'https://uploads.example.com/put', + }); + expect(fileService.createFileRecord).toHaveBeenCalledWith( + expect.objectContaining({ + fileType: 'text/csv', + name: 'result.csv', + size: 42, + url: expect.stringMatching( + /^code-interpreter-exports\/\d{4}-\d{2}-\d{2}\/topic-1\/result\.csv$/, + ), + }), + ); + }); + + it('normalizes provider export failures before storage metadata is read', async () => { + const provider = { + capabilities: { + backgroundCommands: true, + exportFile: true, + files: true, + languages: ['python'], + persistentSession: true, + shell: true, + skillScripts: true, + }, + callTool: vi.fn(), + exportFileToUploadUrl: vi.fn(async () => ({ + error: { message: 'no such file', name: 'not_found' }, + success: false, + })), + kind: 'onlyboxes', + } satisfies SandboxProvider; + + const { SandboxMiddlewareService } = await import('../service'); + const fileService = { + createFileRecord: vi.fn(), + createPreSignedUpload: vi.fn(async () => ({ url: 'https://uploads.example.com/put' })), + getFileMetadata: vi.fn(), + } as unknown as FileService; + + const service = new SandboxMiddlewareService(provider, { + fileService, + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await service.exportAndUploadFile('/workspace/missing.txt', 'missing.txt'); + + expect(result).toMatchObject({ + error: { message: 'no such file', name: 'not_found' }, + filename: 'missing.txt', + success: false, + }); + expect(fileService.getFileMetadata).not.toHaveBeenCalled(); + expect(fileService.createFileRecord).not.toHaveBeenCalled(); + }); +}); diff --git a/src/server/services/sandbox/factory.ts b/src/server/services/sandbox/factory.ts new file mode 100644 index 0000000000..d735e9f068 --- /dev/null +++ b/src/server/services/sandbox/factory.ts @@ -0,0 +1,31 @@ +import { sandboxEnv } from '@/envs/sandbox'; + +import { MarketSandboxProvider } from './providers/market'; +import { OnlyboxesSandboxProvider } from './providers/onlyboxes'; +import { SandboxMiddlewareService } from './service'; +import type { + SandboxProvider, + SandboxProviderKind, + SandboxService, + SandboxServiceOptions, +} from './types'; + +export const getSandboxProviderKind = (): SandboxProviderKind => { + return sandboxEnv.SANDBOX_PROVIDER || 'market'; +}; + +const createSandboxProvider = (options: SandboxServiceOptions): SandboxProvider => { + switch (getSandboxProviderKind()) { + case 'onlyboxes': { + return new OnlyboxesSandboxProvider(options); + } + + case 'market': { + return new MarketSandboxProvider(options); + } + } +}; + +export const createSandboxService = (options: SandboxServiceOptions): SandboxService => { + return new SandboxMiddlewareService(createSandboxProvider(options), options); +}; diff --git a/src/server/services/sandbox/index.ts b/src/server/services/sandbox/index.ts index ea6d2177f0..9031b0440e 100644 --- a/src/server/services/sandbox/index.ts +++ b/src/server/services/sandbox/index.ts @@ -1,186 +1,12 @@ -import { - type ISandboxService, - type SandboxCallToolResult, - type SandboxExportFileResult, -} from '@lobechat/builtin-tool-cloud-sandbox'; -import { type CodeInterpreterToolName } from '@lobehub/market-sdk'; -import debug from 'debug'; -import { sha256 } from 'js-sha256'; - -import { FileS3 } from '@/server/modules/S3'; -import { type FileService } from '@/server/services/file'; -import { type MarketService } from '@/server/services/market'; - -const log = debug('lobe-server:sandbox-service'); - -export interface ServerSandboxServiceOptions { - fileService: FileService; - marketService: MarketService; - topicId: string; - userId: string; -} - -/** - * Server-side Sandbox Service - * - * This service implements ISandboxService for server-side execution. - * Context (topicId, userId) is bound at construction time. - * It uses MarketService to call sandbox tools. - * - * Usage: - * - Used by BuiltinToolsExecutor when executing CloudSandbox tools on server - * - MarketService handles authentication via trustedClientToken - */ -export class ServerSandboxService implements ISandboxService { - private fileService: FileService; - private marketService: MarketService; - private topicId: string; - private userId: string; - - constructor(options: ServerSandboxServiceOptions) { - this.fileService = options.fileService; - this.marketService = options.marketService; - this.topicId = options.topicId; - this.userId = options.userId; - } - - /** - * Call a sandbox tool via MarketService - */ - async callTool(toolName: string, params: Record): Promise { - log('Calling sandbox tool: %s with params: %O, topicId: %s', toolName, params, this.topicId); - - try { - const response = await this.marketService - .getSDK() - .plugins.runBuildInTool(toolName as CodeInterpreterToolName, params as any, { - topicId: this.topicId, - userId: this.userId, - }); - - log('Sandbox tool %s response: %O', toolName, response); - - if (!response.success) { - return { - error: { - message: response.error?.message || 'Unknown error', - name: response.error?.code, - }, - result: null, - sessionExpiredAndRecreated: false, - success: false, - }; - } - - return { - result: response.data?.result, - sessionExpiredAndRecreated: response.data?.sessionExpiredAndRecreated || false, - success: true, - }; - } catch (error) { - log('Error calling sandbox tool %s: %O', toolName, error); - - return { - error: { - message: (error as Error).message, - name: (error as Error).name, - }, - result: null, - sessionExpiredAndRecreated: false, - success: false, - }; - } - } - - /** - * Export and upload a file from sandbox to S3 - * - * Steps: - * 1. Generate S3 pre-signed upload URL - * 2. Call sandbox exportFile tool to upload file - * 3. Verify upload success and get metadata - * 4. Create persistent file record - */ - async exportAndUploadFile(path: string, filename: string): Promise { - log('Exporting file: %s from path: %s, topicId: %s', filename, path, this.topicId); - - try { - const s3 = new FileS3(); - - // Use date-based sharding for privacy compliance (GDPR, CCPA) - const today = new Date().toISOString().split('T')[0]; - - // Generate a unique key for the exported file - const key = `code-interpreter-exports/${today}/${this.topicId}/${filename}`; - - // Step 1: Generate pre-signed upload URL - const uploadUrl = await s3.createPreSignedUrl(key); - log('Generated upload URL for key: %s', key); - - // Step 2: Call sandbox's exportFile tool with the upload URL - const response = await this.marketService.exportFile({ - path, - topicId: this.topicId, - uploadUrl, - userId: this.userId, - }); - - log('Sandbox exportFile response: %O', response); - - if (!response.success) { - return { - error: { message: response.error?.message || 'Failed to export file from sandbox' }, - filename, - success: false, - }; - } - - const result = response.data?.result; - const uploadSuccess = result?.success !== false; - - if (!uploadSuccess) { - return { - error: { message: result?.error || 'Failed to upload file from sandbox' }, - filename, - success: false, - }; - } - - // Step 3: Get file metadata from S3 to verify upload and get actual size - const metadata = await s3.getFileMetadata(key); - const fileSize = metadata.contentLength; - const mimeType = metadata.contentType || result?.mimeType || 'application/octet-stream'; - - // Step 4: Create persistent file record using FileService - // Generate a simple hash from the key (since we don't have the actual file content) - const fileHash = sha256(key + Date.now().toString()); - - const { fileId, url } = await this.fileService.createFileRecord({ - fileHash, - fileType: mimeType, - name: filename, - size: fileSize, - url: key, // Store S3 key - }); - - log('Created file record: fileId=%s, url=%s', fileId, url); - - return { - fileId, - filename, - mimeType, - size: fileSize, - success: true, - url, // This is the permanent /f/:id URL - }; - } catch (error) { - log('Error exporting file: %O', error); - - return { - error: { message: (error as Error).message }, - filename, - success: false, - }; - } - } -} +export { createSandboxService, getSandboxProviderKind } from './factory'; +export { MarketSandboxProvider, ServerSandboxService } from './providers/market'; +export { OnlyboxesSandboxProvider } from './providers/onlyboxes'; +export { normalizeSandboxCommandResult, SandboxMiddlewareService } from './service'; +export type { + SandboxFileExporter, + SandboxProvider, + SandboxProviderKind, + SandboxService, + SandboxServiceOptions, + SandboxSessionContext, +} from './types'; diff --git a/src/server/services/sandbox/providers/market.test.ts b/src/server/services/sandbox/providers/market.test.ts new file mode 100644 index 0000000000..a8caebfa64 --- /dev/null +++ b/src/server/services/sandbox/providers/market.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { MarketService } from '@/server/services/market'; + +import { MarketSandboxProvider, redactSandboxParams } from './market'; + +describe('MarketSandboxProvider', () => { + const createMarketService = (response: unknown) => + ({ + exportFile: vi.fn(async () => response), + getSDK: vi.fn(() => ({ + plugins: { + runBuildInTool: vi.fn(async () => response), + }, + })), + }) as unknown as MarketService; + + it('keeps the previous Market sandbox callTool success response shape', async () => { + const marketService = createMarketService({ + data: { + result: { + exitCode: 0, + stdout: 'ok', + }, + sessionExpiredAndRecreated: true, + }, + success: true, + }); + const provider = new MarketSandboxProvider({ + marketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('runCommand', { command: 'echo ok' }); + + expect(result).toEqual({ + result: { + exitCode: 0, + stdout: 'ok', + }, + sessionExpiredAndRecreated: true, + success: true, + }); + }); + + it('keeps the previous Market sandbox callTool error mapping', async () => { + const marketService = createMarketService({ + error: { + code: 'token_expired', + message: 'expired', + }, + success: false, + }); + const provider = new MarketSandboxProvider({ + marketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('runCommand', { command: 'echo ok' }); + + expect(result).toEqual({ + error: { + message: 'expired', + name: 'token_expired', + }, + result: null, + sessionExpiredAndRecreated: false, + success: false, + }); + }); + + it('preserves Market sandbox export error codes for authorization handling', async () => { + const marketService = createMarketService({ + error: { + code: 'token_expired', + message: 'expired', + }, + success: false, + }); + const provider = new MarketSandboxProvider({ + marketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.exportFileToUploadUrl({ + filename: 'report.txt', + path: '/workspace/report.txt', + uploadUrl: 'https://uploads.example.com/put', + }); + + expect(result).toEqual({ + error: { + message: 'expired', + name: 'token_expired', + }, + success: false, + }); + }); + + it('keeps the previous Market sandbox export success response shape', async () => { + const marketService = createMarketService({ + data: { + result: { + mimeType: 'text/plain', + success: true, + }, + }, + success: true, + }); + const provider = new MarketSandboxProvider({ + marketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.exportFileToUploadUrl({ + filename: 'report.txt', + path: '/workspace/report.txt', + uploadUrl: 'https://uploads.example.com/put', + }); + + expect(result).toEqual({ + mimeType: 'text/plain', + result: { + mimeType: 'text/plain', + success: true, + }, + success: true, + }); + }); + + it('keeps the previous Market sandbox upload failure mapping', async () => { + const marketService = createMarketService({ + data: { + result: { + error: 'upload failed', + success: false, + }, + }, + success: true, + }); + const provider = new MarketSandboxProvider({ + marketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.exportFileToUploadUrl({ + filename: 'report.txt', + path: '/workspace/report.txt', + uploadUrl: 'https://uploads.example.com/put', + }); + + expect(result).toEqual({ + error: { + message: 'upload failed', + }, + success: false, + }); + }); + + describe('redactSandboxParams', () => { + it('redacts auth env assignments from command logs without changing other params', () => { + const params = { + command: + 'LOBEHUB_JWT=mock-jwt LOBEHUB_SERVER=https://app.lobehub.com npx -y @lobehub/cli topic list && GITHUB_TOKEN="ghp_token" gh repo view', + timeout: 1000, + }; + + expect(redactSandboxParams(params)).toEqual({ + command: + 'LOBEHUB_JWT=[redacted] LOBEHUB_SERVER=https://app.lobehub.com npx -y @lobehub/cli topic list && GITHUB_TOKEN=[redacted] gh repo view', + timeout: 1000, + }); + }); + + it('redacts sandbox resource URLs from params', () => { + expect( + redactSandboxParams({ + skillZipUrls: { chart: 'https://files.example.com/chart.zip' }, + zipUrl: 'https://files.example.com/legacy.zip', + }), + ).toEqual({ + skillZipUrls: '[redacted]', + zipUrl: '[redacted]', + }); + }); + }); +}); diff --git a/src/server/services/sandbox/providers/market.ts b/src/server/services/sandbox/providers/market.ts new file mode 100644 index 0000000000..78a4395399 --- /dev/null +++ b/src/server/services/sandbox/providers/market.ts @@ -0,0 +1,170 @@ +import type { SandboxCallToolResult } from '@lobechat/builtin-tool-cloud-sandbox'; +import type { CodeInterpreterToolName } from '@lobehub/market-sdk'; +import debug from 'debug'; + +import { SandboxMiddlewareService } from '../service'; +import type { + SandboxProvider, + SandboxProviderCapabilities, + SandboxProviderFileExportRequest, + SandboxProviderFileExportResult, + SandboxService, + SandboxServiceOptions, +} from '../types'; + +const log = debug('lobe-server:sandbox:market'); +const REDACTED_SANDBOX_PARAM = '[redacted]'; +const SANDBOX_AUTH_ENV_PATTERN = /\b(LOBEHUB_JWT|GITHUB_TOKEN)=("[^"]*"|'[^']*'|\S+)/g; + +export class MarketSandboxProvider implements SandboxProvider { + readonly capabilities = { + backgroundCommands: true, + exportFile: true, + files: true, + languages: ['python', 'javascript', 'typescript'], + persistentSession: true, + shell: true, + skillScripts: true, + } as const satisfies SandboxProviderCapabilities; + + readonly kind = 'market'; + + private readonly options: SandboxServiceOptions; + + constructor(options: SandboxServiceOptions) { + this.options = options; + } + + async callTool( + toolName: string, + params: Record, + ): Promise { + const { marketService, topicId, userId } = this.options; + + log( + 'Calling sandbox tool: %s with params: %O, topicId: %s', + toolName, + redactSandboxParams(params), + topicId, + ); + + try { + const response = await marketService + .getSDK() + .plugins.runBuildInTool(toolName as CodeInterpreterToolName, params as never, { + topicId, + userId, + }); + + log('Sandbox tool %s response: %O', toolName, response); + + if (!response.success) { + return { + error: { + message: response.error?.message || 'Unknown error', + name: response.error?.code, + }, + result: null, + sessionExpiredAndRecreated: false, + success: false, + }; + } + + return { + result: response.data?.result, + sessionExpiredAndRecreated: response.data?.sessionExpiredAndRecreated || false, + success: true, + }; + } catch (error) { + log('Error calling sandbox tool %s: %O', toolName, error); + + return { + error: { + message: (error as Error).message, + name: (error as Error).name, + }, + result: null, + sessionExpiredAndRecreated: false, + success: false, + }; + } + } + + async exportFileToUploadUrl({ + path, + uploadUrl, + }: SandboxProviderFileExportRequest): Promise { + const { marketService, topicId, userId } = this.options; + + try { + const response = await marketService.exportFile({ + path, + topicId, + uploadUrl, + userId, + }); + + log('Sandbox exportFile response: %O', response); + + if (!response.success) { + return { + error: { + message: response.error?.message || 'Failed to export file from sandbox', + name: response.error?.code, + }, + success: false, + }; + } + + const result = response.data?.result; + const uploadSuccess = result?.success !== false; + + if (!uploadSuccess) { + return { + error: { message: result?.error || 'Failed to upload file from sandbox' }, + success: false, + }; + } + + return { + mimeType: result?.mimeType, + result, + success: true, + }; + } catch (error) { + log('Error exporting file: %O', error); + + return { + error: { message: (error as Error).message }, + success: false, + }; + } + } +} + +export const redactSandboxParams = (params: Record) => { + const hasCommand = typeof params.command === 'string'; + if (!params.skillZipUrls && !params.zipUrl && !hasCommand) return params; + + const redacted = { + ...params, + }; + + if (params.zipUrl) redacted.zipUrl = REDACTED_SANDBOX_PARAM; + if (params.skillZipUrls) redacted.skillZipUrls = REDACTED_SANDBOX_PARAM; + if (typeof params.command === 'string') { + redacted.command = params.command.replaceAll( + SANDBOX_AUTH_ENV_PATTERN, + (_, name: string) => `${name}=${REDACTED_SANDBOX_PARAM}`, + ); + } + + return redacted; +}; + +/** @deprecated Use createSandboxService. */ +export class ServerSandboxService extends SandboxMiddlewareService implements SandboxService { + constructor(options: SandboxServiceOptions) { + super(new MarketSandboxProvider(options), options); + } +} diff --git a/src/server/services/sandbox/providers/onlyboxes.test.ts b/src/server/services/sandbox/providers/onlyboxes.test.ts new file mode 100644 index 0000000000..3983f1df1a --- /dev/null +++ b/src/server/services/sandbox/providers/onlyboxes.test.ts @@ -0,0 +1,554 @@ +import { createHmac } from 'node:crypto'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MarketService } from '@/server/services/market'; + +const decodeJITPayload = (authorization?: string) => { + const token = authorization?.replace('Bearer ', '') || ''; + const [payload] = token.replace('obx_jit_v1.', '').split('.'); + + return JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as { + exp: number; + iss: string; + sub: string; + }; +}; + +const verifyJITSignature = (authorization?: string) => { + const token = authorization?.replace('Bearer ', '') || ''; + const [signed, signature] = token.split(/\.(?=[^.]+$)/); + const expected = createHmac('sha256', 'jit-signing-key').update(signed).digest('base64url'); + + return signature === expected; +}; + +describe('OnlyboxesSandboxProvider', () => { + beforeEach(() => { + vi.resetModules(); + vi.restoreAllMocks(); + vi.setSystemTime(new Date('2026-05-30T00:00:00.000Z')); + vi.doMock('@/envs/app', () => ({ + appEnv: { + APP_URL: 'https://lobehub.example.com', + }, + })); + vi.doMock('@/envs/sandbox', () => ({ + sandboxEnv: { + ONLYBOXES_BASE_URL: 'https://onlyboxes.example.com/', + ONLYBOXES_JIT_SIGNING_KEY: 'jit-signing-key', + ONLYBOXES_JIT_TTL_SEC: 900, + ONLYBOXES_LEASE_TTL_SEC: 120, + }, + })); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('maps runCommand to the terminal command endpoint with a persistent session', async () => { + const fetchMock = vi.fn(async () => { + return new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: 'ok\n', + }), + { status: 200 }, + ); + }); + vi.stubGlobal('fetch', fetchMock); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('runCommand', { command: 'echo ok' }); + + expect(result).toMatchObject({ + result: { exitCode: 0, stdout: 'ok\n' }, + success: true, + }); + expect(fetchMock).toHaveBeenCalledWith( + 'https://onlyboxes.example.com/api/v1/commands/terminal', + expect.objectContaining({ + body: JSON.stringify({ + command: 'echo ok', + create_if_missing: true, + lease_ttl_sec: 120, + session_id: 'lobe-user-1-topic-1', + timeout_ms: 120_000, + }), + method: 'POST', + }), + ); + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + const headers = init.headers as Headers; + const authorization = headers.get('Authorization') || undefined; + expect(authorization).toMatch(/^Bearer obx_jit_v1\./); + expect(verifyJITSignature(authorization)).toBe(true); + expect(decodeJITPayload(authorization)).toEqual({ + exp: Date.parse('2026-05-30T00:15:00.000Z'), + iss: 'https://lobehub.example.com', + sub: 'user-1', + }); + }); + + it('treats non-zero terminal exit codes as successful tool transport results', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return new Response( + JSON.stringify({ + exit_code: 2, + session_id: 'lobe-user-1-topic-1', + stderr: 'failed\n', + stdout: 'partial\n', + }), + { status: 200 }, + ); + }), + ); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('runCommand', { command: 'exit 2' }); + + expect(result).toMatchObject({ + result: { + exitCode: 2, + stderr: 'failed\n', + stdout: 'partial\n', + success: false, + }, + success: true, + }); + }); + + it('returns a provider error when background command submission fails', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return new Response( + JSON.stringify({ + error: { + code: 'no_worker', + message: 'no compatible worker', + }, + status: 'failed', + }), + { status: 200 }, + ); + }), + ); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('runCommand', { + background: true, + command: 'sleep 10', + }); + + expect(result).toMatchObject({ + error: { message: 'no compatible worker' }, + result: null, + success: false, + }); + }); + + it('treats running background command polls as successful output retrievals', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return new Response( + JSON.stringify({ + result: { + stdout: 'partial\n', + }, + status: 'running', + task_id: 'task-1', + }), + { status: 200 }, + ); + }), + ); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('getCommandOutput', { commandId: 'task-1' }); + + expect(result).toMatchObject({ + result: { + newOutput: 'partial\n', + running: true, + success: true, + }, + success: true, + }); + }); + + it('unwraps JSON output from terminal-backed file operations', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: JSON.stringify({ + files: [{ isDirectory: false, name: 'a.txt' }], + totalCount: 1, + }), + }), + { status: 200 }, + ); + }), + ); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('listLocalFiles', { directoryPath: '/workspace' }); + + expect(result).toMatchObject({ + result: { + files: [{ isDirectory: false, name: 'a.txt' }], + totalCount: 1, + }, + success: true, + }); + }); + + it('writes files through chunked terminal scripts instead of embedding content in one command', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: JSON.stringify({ success: true }), + }), + { status: 200 }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: JSON.stringify({ bytesWritten: 11, success: true }), + }), + { status: 200 }, + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('writeLocalFile', { + content: 'hello world', + createDirectories: true, + path: '/workspace/report.txt', + }); + + expect(result).toMatchObject({ + result: { bytesWritten: 11, success: true }, + success: true, + }); + const firstCallBody = JSON.parse(String(fetchMock.mock.calls[0][1].body)) as { + command: string; + }; + const secondCallBody = JSON.parse(String(fetchMock.mock.calls[1][1].body)) as { + command: string; + }; + expect(firstCallBody.command).toContain("path.write_bytes(b'')"); + expect(secondCallBody.command).toContain("path.open('ab')"); + expect(firstCallBody.command).not.toContain('hello world'); + expect(secondCallBody.command).not.toContain('hello world'); + }); + + it('ensures a terminal session exists before exporting files through terminalResource', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: '', + }), + { status: 200 }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + result: { + file_path: '/workspace/report.txt', + mime_type: 'text/plain', + session_id: 'lobe-user-1-topic-1', + size_bytes: 12, + }, + status: 'succeeded', + task_id: 'task-1', + }), + { status: 200 }, + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.exportFileToUploadUrl({ + filename: 'report.txt', + path: '/workspace/report.txt', + uploadHeaders: { 'x-amz-acl': 'public-read' }, + uploadUrl: 'https://uploads.example.com/put', + }); + + expect(result).toMatchObject({ + mimeType: 'text/plain', + success: true, + }); + expect(fetchMock).toHaveBeenNthCalledWith( + 1, + 'https://onlyboxes.example.com/api/v1/commands/terminal', + expect.any(Object), + ); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + 'https://onlyboxes.example.com/api/v1/tasks', + expect.objectContaining({ + body: JSON.stringify({ + capability: 'terminalResource', + input: { + action: 'export', + file_path: '/workspace/report.txt', + headers: { 'x-amz-acl': 'public-read' }, + session_id: 'lobe-user-1-topic-1', + signed_url: 'https://uploads.example.com/put', + }, + mode: 'sync', + timeout_ms: 120_000, + wait_ms: 60_000, + }), + }), + ); + }); + + it('runs execScript from a prepared skill directory when skill zip URLs are available', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: '', + }), + { status: 200 }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: 'from skill\n', + }), + { status: 200 }, + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('execScript', { + activatedSkills: [{ id: 'skill-1', name: 'demo' }], + command: 'python scripts/run.py', + skillZipUrls: { demo: 'https://files.example.com/demo.zip' }, + }); + + expect(result).toMatchObject({ + result: { + stdout: 'from skill\n', + success: true, + }, + success: true, + }); + + const setupBody = JSON.parse(String(fetchMock.mock.calls[0][1].body)) as { command: string }; + const commandBody = JSON.parse(String(fetchMock.mock.calls[1][1].body)) as { command: string }; + expect(setupBody.command).toContain("curl -fsSL 'https://files.example.com/demo.zip'"); + expect(setupBody.command).toContain('unzip -q'); + expect(commandBody.command).toContain("cd '/tmp/lobe-skills/"); + expect(commandBody.command).toContain("/demo'"); + expect(commandBody.command).toContain('python scripts/run.py'); + }); + + it('prepares all skill zip URLs and runs execScript from the last activated skill directory', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: '', + }), + { status: 200 }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: 'from second skill\n', + }), + { status: 200 }, + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('execScript', { + activatedSkills: [ + { id: 'skill-1', name: 'first skill' }, + { id: 'skill-2', name: 'second/skill' }, + ], + command: 'python scripts/run.py', + skillZipUrls: { + 'first skill': 'https://files.example.com/first.zip', + 'second/skill': 'https://files.example.com/second.zip', + }, + }); + + expect(result).toMatchObject({ + result: { + stdout: 'from second skill\n', + success: true, + }, + success: true, + }); + + const setupBody = JSON.parse(String(fetchMock.mock.calls[0][1].body)) as { command: string }; + const commandBody = JSON.parse(String(fetchMock.mock.calls[1][1].body)) as { command: string }; + expect(setupBody.command).toContain("curl -fsSL 'https://files.example.com/first.zip'"); + expect(setupBody.command).toContain("curl -fsSL 'https://files.example.com/second.zip'"); + expect(setupBody.command).toContain('/first-skill/'); + expect(setupBody.command).toContain('/second-skill/'); + expect(commandBody.command).toContain("cd '/tmp/lobe-skills/"); + expect(commandBody.command).toContain("/second-skill'"); + expect(commandBody.command).toContain('python scripts/run.py'); + }); + + it('uses the configured skill name for legacy single zipUrl execScript calls', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: '', + }), + { status: 200 }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + exit_code: 0, + session_id: 'lobe-user-1-topic-1', + stderr: '', + stdout: 'from legacy skill\n', + }), + { status: 200 }, + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const { OnlyboxesSandboxProvider } = await import('./onlyboxes'); + const provider = new OnlyboxesSandboxProvider({ + marketService: {} as MarketService, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await provider.callTool('execScript', { + command: 'python scripts/run.py', + config: { name: 'legacy skill' }, + zipUrl: 'https://files.example.com/legacy.zip', + }); + + expect(result).toMatchObject({ + result: { + stdout: 'from legacy skill\n', + success: true, + }, + success: true, + }); + + const setupBody = JSON.parse(String(fetchMock.mock.calls[0][1].body)) as { command: string }; + const commandBody = JSON.parse(String(fetchMock.mock.calls[1][1].body)) as { command: string }; + expect(setupBody.command).toContain("curl -fsSL 'https://files.example.com/legacy.zip'"); + expect(setupBody.command).toContain('/legacy-skill/'); + expect(commandBody.command).toContain('/legacy-skill'); + }); +}); diff --git a/src/server/services/sandbox/providers/onlyboxes.ts b/src/server/services/sandbox/providers/onlyboxes.ts new file mode 100644 index 0000000000..3f9737f001 --- /dev/null +++ b/src/server/services/sandbox/providers/onlyboxes.ts @@ -0,0 +1,901 @@ +import { createHmac } from 'node:crypto'; + +import type { SandboxCallToolResult } from '@lobechat/builtin-tool-cloud-sandbox'; +import { isRecord } from '@lobechat/utils'; +import debug from 'debug'; +import { sha256 } from 'js-sha256'; + +import { appEnv } from '@/envs/app'; +import { sandboxEnv } from '@/envs/sandbox'; + +import type { + SandboxProvider, + SandboxProviderCapabilities, + SandboxProviderFileExportRequest, + SandboxProviderFileExportResult, + SandboxServiceOptions, +} from '../types'; + +const log = debug('lobe-server:sandbox:onlyboxes'); + +const DEFAULT_TIMEOUT_MS = 120_000; +const EXPORT_TASK_WAIT_MS = 60_000; +const DEFAULT_LEASE_TTL_SEC = 900; +const DEFAULT_JIT_TTL_SEC = 1800; +const JIT_TOKEN_PREFIX = 'obx_jit_v1.'; +const WRITE_FILE_CHUNK_BYTES = 48 * 1024; +const SKILL_ARCHIVE_CACHE_DIR = '/tmp/lobe-skills'; + +interface OnlyboxesTaskResponse { + error?: { code?: string; message?: string }; + result?: Record; + status?: string; + task_id?: string; +} + +interface TerminalExecResult { + created?: boolean; + exit_code?: number; + lease_expires_unix_ms?: number; + session_id?: string; + stderr?: string; + stderr_truncated?: boolean; + stdout?: string; + stdout_truncated?: boolean; +} + +export class OnlyboxesSandboxProvider implements SandboxProvider { + readonly capabilities = { + backgroundCommands: true, + exportFile: true, + files: true, + languages: ['python', 'javascript', 'typescript'], + persistentSession: true, + shell: true, + skillScripts: true, + } as const satisfies SandboxProviderCapabilities; + + readonly kind = 'onlyboxes'; + + private readonly baseUrl: string; + private readonly jitIssuer: string; + private readonly jitSigningKey: string; + private readonly jitTTLSec: number; + private readonly leaseTTLSec: number; + private readonly options: SandboxServiceOptions; + + constructor(options: SandboxServiceOptions) { + this.options = options; + this.baseUrl = (sandboxEnv.ONLYBOXES_BASE_URL || '').replace(/\/+$/, ''); + this.jitIssuer = sandboxEnv.ONLYBOXES_JIT_ISSUER || appEnv.APP_URL || 'lobehub'; + this.jitSigningKey = sandboxEnv.ONLYBOXES_JIT_SIGNING_KEY || ''; + this.jitTTLSec = sandboxEnv.ONLYBOXES_JIT_TTL_SEC || DEFAULT_JIT_TTL_SEC; + this.leaseTTLSec = sandboxEnv.ONLYBOXES_LEASE_TTL_SEC || DEFAULT_LEASE_TTL_SEC; + } + + async callTool( + toolName: string, + params: Record, + ): Promise { + if (!this.baseUrl || !this.jitSigningKey) { + return this.errorResult('ONLYBOXES_BASE_URL and ONLYBOXES_JIT_SIGNING_KEY are required'); + } + + try { + switch (toolName) { + case 'runCommand': { + return this.runCommand(params); + } + + case 'getCommandOutput': { + return this.getCommandOutput(params); + } + + case 'killCommand': { + return this.killCommand(params); + } + + case 'executeCode': { + return this.executeCode(params); + } + + case 'execScript': { + return this.execScript(params); + } + + case 'listLocalFiles': { + return this.runJsonScript(listFilesScript, params); + } + + case 'listFiles': { + return this.runJsonScript(listFilesScript, params); + } + + case 'readLocalFile': { + return this.runJsonScript(readFileScript, params); + } + + case 'readFile': { + return this.runJsonScript(readFileScript, params); + } + + case 'writeLocalFile': { + return this.writeLocalFile(params); + } + + case 'writeFile': { + return this.writeLocalFile(params); + } + + case 'editLocalFile': { + return this.runJsonScript(editFileScript, params); + } + + case 'editFile': { + return this.runJsonScript(editFileScript, params); + } + + case 'searchLocalFiles': { + return this.runJsonScript(searchFilesScript, params); + } + + case 'searchFiles': { + return this.runJsonScript(searchFilesScript, params); + } + + case 'moveLocalFiles': { + return this.runJsonScript(moveFilesScript, params); + } + + case 'moveFiles': { + return this.runJsonScript(moveFilesScript, params); + } + + case 'grepContent': { + return this.runJsonScript(grepContentScript, params); + } + + case 'globLocalFiles': { + return this.runJsonScript(globFilesScript, params); + } + + case 'globFiles': { + return this.runJsonScript(globFilesScript, params); + } + + default: { + return this.errorResult(`Unsupported Onlyboxes sandbox tool: ${toolName}`); + } + } + } catch (error) { + log('Onlyboxes tool %s failed: %O', toolName, error); + return this.errorResult((error as Error).message, (error as Error).name); + } + } + + async exportFileToUploadUrl({ + path, + uploadHeaders, + uploadUrl, + }: SandboxProviderFileExportRequest): Promise { + if (!this.baseUrl || !this.jitSigningKey) { + return { + error: { message: 'ONLYBOXES_BASE_URL and ONLYBOXES_JIT_SIGNING_KEY are required' }, + success: false, + }; + } + + try { + await this.ensureSession(); + + const task = await this.submitTask('terminalResource', { + action: 'export', + file_path: path, + headers: uploadHeaders, + session_id: this.sessionId, + signed_url: uploadUrl, + }); + + if (task.status !== 'succeeded') { + return { + error: { message: task.error?.message || 'Failed to export file from Onlyboxes sandbox' }, + success: false, + }; + } + + return { + mimeType: String(task.result?.mime_type || ''), + result: task.result, + size: typeof task.result?.size_bytes === 'number' ? task.result.size_bytes : undefined, + success: true, + }; + } catch (error) { + log('Onlyboxes export failed: %O', error); + return { + error: { message: (error as Error).message }, + success: false, + }; + } + } + + private get sessionId() { + const scope = `${this.options.userId}-${this.options.topicId}`; + return `lobe-${scope.replaceAll(/[^\w.-]/g, '-')}`; + } + + private async executeCode(params: Record): Promise { + const code = String(params.code || ''); + const language = String(params.language || 'python'); + + const runners: Record = { + javascript: 'node', + python: 'python3', + typescript: 'npx --yes tsx', + }; + const extensions: Record = { + javascript: 'js', + python: 'py', + typescript: 'ts', + }; + const runner = runners[language]; + + if (!runner) { + return this.errorResult(`Unsupported code language for Onlyboxes sandbox: ${language}`); + } + + const filePath = `/tmp/lobe-code-${Date.now()}.${extensions[language]}`; + const writeResult = await this.writeTextFile({ + content: code, + createDirectories: true, + path: filePath, + timeoutMs: this.timeout(params), + }); + + if (!writeResult.success) { + return writeResult; + } + + const command = `${runner} '${filePath}'`; + const terminal = await this.execTerminal(command, this.timeout(params)); + + return { + result: { + error: terminal.exit_code === 0 ? undefined : terminal.stderr, + exitCode: terminal.exit_code, + output: terminal.stdout, + stderr: terminal.stderr, + }, + success: true, + }; + } + + private async execScript(params: Record): Promise { + const command = String(params.command || ''); + + if (!command.trim()) { + return this.errorResult('command is required'); + } + + const skillZipUrls = this.resolveExecScriptZipUrls(params); + const timeoutMs = this.timeout(params); + + if (Object.keys(skillZipUrls).length === 0) { + return this.runCommand({ command, timeout: timeoutMs }); + } + + const defaultSkillName = this.resolveExecScriptSkillName(params, skillZipUrls); + const workspaceDir = this.skillWorkspaceDir(skillZipUrls); + const setupCommand = this.buildSkillSetupCommand({ skillZipUrls, workspaceDir }); + const setup = await this.execTerminal(setupCommand, timeoutMs); + + if (setup.exit_code !== 0) { + return { + error: { message: setup.stderr || setup.stdout || 'Failed to prepare skill resources' }, + result: { + exitCode: setup.exit_code, + output: setup.stdout, + stderr: setup.stderr, + }, + success: false, + }; + } + + const runDir = defaultSkillName + ? `${workspaceDir}/${this.safeSkillDirName(defaultSkillName)}` + : workspaceDir; + const result = await this.execTerminal( + `cd ${this.shellQuote(runDir)} && ${command}`, + timeoutMs, + ); + + return { + result: { + commandId: result.session_id, + exitCode: result.exit_code, + output: result.stdout, + stderr: result.stderr, + stdout: result.stdout, + success: result.exit_code === 0, + }, + success: true, + }; + } + + private async runCommand(params: Record): Promise { + const command = String(params.command || ''); + + if (!command.trim()) { + return this.errorResult('command is required'); + } + + if (params.background === true) { + const task = await this.submitTask( + 'terminalExec', + { + command, + create_if_missing: true, + lease_ttl_sec: this.leaseTTLSec, + session_id: this.sessionId, + }, + { mode: 'async', timeoutMs: this.timeout(params) }, + ); + + if (task.error || !task.task_id) { + return this.errorResult( + task.error?.message || task.error?.code || 'Failed to start Onlyboxes background command', + ); + } + + return { + result: { + commandId: task.task_id, + shell_id: task.task_id, + }, + success: true, + }; + } + + const terminal = await this.execTerminal(command, this.timeout(params)); + + return { + result: { + commandId: terminal.session_id, + exitCode: terminal.exit_code, + output: terminal.stdout, + stderr: terminal.stderr, + stdout: terminal.stdout, + success: terminal.exit_code === 0, + }, + success: true, + }; + } + + private resolveExecScriptZipUrls(params: Record) { + const zipUrl = typeof params.zipUrl === 'string' ? params.zipUrl : undefined; + if (zipUrl) return { [this.resolveLegacyExecScriptSkillName(params)]: zipUrl }; + + if (!isRecord(params.skillZipUrls)) return {}; + + const result: Record = {}; + + for (const [name, value] of Object.entries(params.skillZipUrls)) { + if (typeof value === 'string' && value) { + result[name] = value; + } + } + + return result; + } + + private resolveLegacyExecScriptSkillName(params: Record) { + const configName = isRecord(params.config) ? params.config.name : undefined; + if (typeof configName === 'string' && configName) return configName; + + if (Array.isArray(params.activatedSkills)) { + for (const skill of [...params.activatedSkills].reverse()) { + if (!isRecord(skill)) continue; + + const name = typeof skill.name === 'string' ? skill.name : undefined; + if (name) return name; + } + } + + return 'default'; + } + + private resolveExecScriptSkillName( + params: Record, + skillZipUrls: Record, + ) { + const configName = isRecord(params.config) ? params.config.name : undefined; + if (typeof configName === 'string' && skillZipUrls[configName]) return configName; + + if (Array.isArray(params.activatedSkills)) { + for (const skill of [...params.activatedSkills].reverse()) { + if (!isRecord(skill)) continue; + + const name = typeof skill.name === 'string' ? skill.name : undefined; + if (name && skillZipUrls[name]) return name; + } + } + + const [firstName] = Object.keys(skillZipUrls); + return firstName; + } + + private skillWorkspaceDir(skillZipUrls: Record) { + const entries = Object.entries(skillZipUrls).sort(([left], [right]) => + left.localeCompare(right), + ); + const cacheKey = sha256(JSON.stringify(entries)).slice(0, 32); + return `${SKILL_ARCHIVE_CACHE_DIR}/${cacheKey || 'default'}`; + } + + private buildSkillSetupCommand({ + skillZipUrls, + workspaceDir, + }: { + skillZipUrls: Record; + workspaceDir: string; + }) { + const quotedWorkspaceDir = this.shellQuote(workspaceDir); + const setupCommands = Object.entries(skillZipUrls).map(([name, zipUrl]) => { + const skillDir = `${workspaceDir}/${this.safeSkillDirName(name)}`; + const markerPath = `${skillDir}/.prepared`; + const archivePath = `${skillDir}/skill.zip`; + const quotedArchivePath = this.shellQuote(archivePath); + const quotedDir = this.shellQuote(skillDir); + const quotedMarkerPath = this.shellQuote(markerPath); + const quotedUrl = this.shellQuote(zipUrl); + + return `if [ ! -f ${quotedMarkerPath} ]; then rm -rf ${quotedDir} && mkdir -p ${quotedDir} && curl -fsSL ${quotedUrl} -o ${quotedArchivePath} && unzip -q ${quotedArchivePath} -d ${quotedDir} && printf prepared > ${quotedMarkerPath}; fi`; + }); + + return [ + `mkdir -p ${this.shellQuote(SKILL_ARCHIVE_CACHE_DIR)}`, + `mkdir -p ${quotedWorkspaceDir}`, + ...setupCommands, + ].join(' && '); + } + + private safeSkillDirName(name: string) { + return name.replaceAll(/[^\w.-]/g, '-'); + } + + private shellQuote(value: string) { + return `'${value.replaceAll("'", "'\\''")}'`; + } + + private async writeLocalFile(params: Record): Promise { + const path = String(params.path || ''); + + if (!path) { + return this.errorResult('path is required'); + } + + return this.writeTextFile({ + content: String(params.content || ''), + createDirectories: params.createDirectories === true, + path, + timeoutMs: this.timeout(params), + }); + } + + private async writeTextFile({ + content, + createDirectories, + path, + timeoutMs, + }: { + content: string; + createDirectories: boolean; + path: string; + timeoutMs: number; + }): Promise { + const init = await this.runJsonScript( + prepareWriteFileScript, + { createDirectories, path }, + timeoutMs, + ); + + if (!init.success) { + return init; + } + + const bytes = Buffer.from(content); + let bytesWritten = 0; + + for (let offset = 0; offset < bytes.length; offset += WRITE_FILE_CHUNK_BYTES) { + const chunk = bytes.subarray(offset, offset + WRITE_FILE_CHUNK_BYTES).toString('base64'); + const append = await this.runJsonScript( + appendWriteFileChunkScript, + { chunk, path }, + timeoutMs, + ); + + if (!append.success) { + return append; + } + + bytesWritten += Number(append.result?.bytesWritten || 0); + } + + return { + result: { + bytesWritten, + success: true, + }, + success: true, + }; + } + + private async getCommandOutput(params: Record): Promise { + const commandId = String(params.commandId || ''); + if (!commandId) return this.errorResult('commandId is required'); + + const task = await this.request(`/api/v1/tasks/${commandId}`, { + method: 'GET', + }); + + const running = + task.status === 'running' || task.status === 'pending' || task.status === 'dispatched'; + const success = running || task.status === 'succeeded'; + const result = task.result || {}; + + return { + error: task.error + ? { message: task.error.message || task.error.code || 'Task failed' } + : undefined, + result: { + error: task.error?.message, + newOutput: String(result.stdout || result.output || ''), + output: String(result.stdout || result.output || ''), + running, + stderr: String(result.stderr || ''), + success, + }, + success: !task.error, + }; + } + + private async killCommand(params: Record): Promise { + const commandId = String(params.commandId || ''); + if (!commandId) return this.errorResult('commandId is required'); + + const task = await this.request(`/api/v1/tasks/${commandId}/cancel`, { + method: 'POST', + }); + + return { + error: task.error + ? { message: task.error.message || task.error.code || 'Failed to cancel task' } + : undefined, + result: { + success: !task.error, + }, + success: !task.error, + }; + } + + private async runJsonScript( + script: string, + params: Record, + timeoutMs = this.timeout(params), + ): Promise { + const encoded = Buffer.from(JSON.stringify(params)).toString('base64'); + const command = `python3 - <<'PY'\n${script}\nmain('${encoded}')\nPY`; + const terminal = await this.execTerminal(command, timeoutMs); + + if (terminal.exit_code !== 0) { + return { + error: { message: terminal.stderr || terminal.stdout || 'Onlyboxes script failed' }, + result: null, + success: false, + }; + } + + try { + const result = JSON.parse(terminal.stdout || '{}') as Record; + + if (result.success === false) { + return { + error: { message: String(result.error || 'Onlyboxes script failed') }, + result, + success: false, + }; + } + + return { + result, + success: true, + }; + } catch (error) { + return { + error: { message: `Failed to parse Onlyboxes script output: ${(error as Error).message}` }, + result: { output: terminal.stdout, stderr: terminal.stderr }, + success: false, + }; + } + } + + private async execTerminal(command: string, timeoutMs = DEFAULT_TIMEOUT_MS) { + return this.request('/api/v1/commands/terminal', { + body: JSON.stringify({ + command, + create_if_missing: true, + lease_ttl_sec: this.leaseTTLSec, + session_id: this.sessionId, + timeout_ms: timeoutMs, + }), + method: 'POST', + }); + } + + private async ensureSession() { + await this.execTerminal(':', DEFAULT_TIMEOUT_MS); + } + + private async submitTask( + capability: string, + input: Record, + options?: { mode?: 'async' | 'auto' | 'sync'; timeoutMs?: number }, + ) { + return this.request('/api/v1/tasks', { + body: JSON.stringify({ + capability, + input, + mode: options?.mode || 'sync', + timeout_ms: options?.timeoutMs || DEFAULT_TIMEOUT_MS, + wait_ms: options?.mode === 'async' ? 1 : EXPORT_TASK_WAIT_MS, + }), + method: 'POST', + }); + } + + private async request(path: string, init: RequestInit): Promise { + const headers = new Headers(init.headers); + headers.set('Authorization', `Bearer ${this.createJITToken()}`); + headers.set('Content-Type', 'application/json'); + + const response = await fetch(`${this.baseUrl}${path}`, { + ...init, + headers, + }); + const body = await response.text(); + const json = body ? JSON.parse(body) : {}; + + if (!response.ok) { + const message = + typeof json?.error === 'string' + ? json.error + : typeof json?.error?.message === 'string' + ? json.error.message + : `Onlyboxes request failed with HTTP ${response.status}`; + throw new Error(message); + } + + return json as T; + } + + private createJITToken(now = Date.now()) { + const claims = { + exp: now + this.jitTTLSec * 1000, + iss: this.jitIssuer, + sub: this.options.userId, + }; + const payload = Buffer.from(JSON.stringify(claims)).toString('base64url'); + const signed = `${JIT_TOKEN_PREFIX}${payload}`; + const signature = createHmac('sha256', this.jitSigningKey).update(signed).digest('base64url'); + + return `${signed}.${signature}`; + } + + private timeout(params: Record) { + const value = params.timeout ?? params.timeout_ms; + return typeof value === 'number' && Number.isFinite(value) ? value : DEFAULT_TIMEOUT_MS; + } + + private errorResult(message: string, name?: string): SandboxCallToolResult { + return { + error: { message, name }, + result: null, + success: false, + }; + } +} + +const scriptPrelude = ` +import base64, json, os, re, shutil, glob, fnmatch +from pathlib import Path + +def load_args(encoded): + return json.loads(base64.b64decode(encoded).decode()) + +def emit(value): + print(json.dumps(value, ensure_ascii=False)) +`; + +const listFilesScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + directory = args.get('directoryPath') or '.' + entries = [] + for entry in os.scandir(directory): + stat = entry.stat() + entries.append({ + 'name': entry.name, + 'path': entry.path, + 'isDirectory': entry.is_dir(), + 'size': stat.st_size, + 'mtime': stat.st_mtime, + }) + emit({'files': entries, 'totalCount': len(entries)}) +`; + +const readFileScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + path = args.get('path') + start = args.get('startLine') + end = args.get('endLine') + text = Path(path).read_text(errors='replace') + lines = text.splitlines(True) + selected = lines + if start is not None or end is not None: + start_idx = max((start or 1) - 1, 0) + end_idx = end if end is not None else len(lines) + selected = lines[start_idx:end_idx] + content = ''.join(selected) + emit({ + 'content': content, + 'filename': os.path.basename(path), + 'charCount': len(content), + 'totalCharCount': len(text), + 'totalLineCount': len(lines), + }) +`; + +const prepareWriteFileScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + path = Path(args.get('path')) + if args.get('createDirectories'): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'') + emit({'success': True}) +`; + +const appendWriteFileChunkScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + path = Path(args.get('path')) + chunk = base64.b64decode(args.get('chunk') or '') + with path.open('ab') as file: + file.write(chunk) + emit({'bytesWritten': len(chunk), 'success': True}) +`; + +const editFileScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + path = Path(args.get('path')) + search = args.get('search') or '' + replace = args.get('replace') or '' + text = path.read_text(errors='replace') + count = text.count(search) + if count == 0: + emit({'success': False, 'error': 'search text not found', 'replacements': 0}) + return + new_text = text.replace(search, replace) if args.get('all') else text.replace(search, replace, 1) + replacements = count if args.get('all') else 1 + path.write_text(new_text) + emit({'success': True, 'replacements': replacements, 'linesAdded': replace.count('\\n'), 'linesDeleted': search.count('\\n')}) +`; + +const searchFilesScript = `${scriptPrelude} +from datetime import datetime + +def parse_time(value): + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace('Z', '+00:00')).timestamp() + except Exception: + return None + +def main(encoded): + args = load_args(encoded) + directory = args.get('directory') or '.' + raw_keywords = args.get('keywords') or args.get('keyword') or '' + keywords = [item.strip() for item in str(raw_keywords).split() if item.strip()] + raw_file_types = args.get('fileTypes') or args.get('fileType') or [] + if isinstance(raw_file_types, str): + raw_file_types = [raw_file_types] + file_types = [item if str(item).startswith('.') else f'.{item}' for item in raw_file_types if str(item).strip()] + modified_after = parse_time(args.get('modifiedAfter')) + modified_before = parse_time(args.get('modifiedBefore')) + content_contains = args.get('contentContains') + limit = args.get('limit') + results = [] + for root, _, files in os.walk(directory): + for name in files: + if keywords and not all(keyword in name for keyword in keywords): + continue + if file_types and not any(name.endswith(file_type) for file_type in file_types): + continue + path = os.path.join(root, name) + try: + stat = os.stat(path) + except Exception: + continue + if modified_after is not None and stat.st_mtime < modified_after: + continue + if modified_before is not None and stat.st_mtime > modified_before: + continue + if content_contains: + try: + if str(content_contains) not in Path(path).read_text(errors='replace'): + continue + except Exception: + continue + results.append({'name': name, 'path': path, 'size': stat.st_size, 'mtime': stat.st_mtime}) + sort_by = args.get('sortBy') + reverse = args.get('sortDirection') == 'desc' + if sort_by == 'size': + results.sort(key=lambda item: item.get('size') or 0, reverse=reverse) + elif sort_by == 'date': + results.sort(key=lambda item: item.get('mtime') or 0, reverse=reverse) + else: + results.sort(key=lambda item: item.get('name') or '', reverse=reverse) + total = len(results) + if isinstance(limit, int) and limit > 0: + results = results[:limit] + emit({'results': results, 'totalCount': total}) +`; + +const moveFilesScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + results = [] + for op in args.get('operations') or []: + try: + shutil.move(op.get('source'), op.get('destination')) + results.append({'source': op.get('source'), 'destination': op.get('destination'), 'success': True}) + except Exception as error: + results.append({'source': op.get('source'), 'destination': op.get('destination'), 'success': False, 'error': str(error)}) + emit({'results': results, 'successCount': len([r for r in results if r.get('success')])}) +`; + +const grepContentScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + directory = args.get('directory') or '.' + pattern = args.get('pattern') or '' + file_pattern = args.get('filePattern') or '*' + recursive = args.get('recursive', True) + regex = re.compile(pattern) + matches = [] + walker = os.walk(directory) if recursive else [(directory, [], os.listdir(directory))] + for root, _, files in walker: + for name in files: + if not fnmatch.fnmatch(name, file_pattern): + continue + path = os.path.join(root, name) + try: + with open(path, 'r', errors='replace') as file: + for index, line in enumerate(file, 1): + if regex.search(line): + matches.append({'path': path, 'lineNumber': index, 'line': line.rstrip('\\n')}) + except Exception: + pass + emit({'matches': matches, 'totalMatches': len(matches)}) +`; + +const globFilesScript = `${scriptPrelude} +def main(encoded): + args = load_args(encoded) + directory = args.get('directory') or '.' + pattern = args.get('pattern') or '*' + files = glob.glob(os.path.join(directory, pattern), recursive=True) + emit({'files': files, 'totalCount': len(files)}) +`; diff --git a/src/server/services/sandbox/service.ts b/src/server/services/sandbox/service.ts new file mode 100644 index 0000000000..cc97e5d282 --- /dev/null +++ b/src/server/services/sandbox/service.ts @@ -0,0 +1,135 @@ +import type { + SandboxCallToolResult, + SandboxExportFileResult, +} from '@lobechat/builtin-tool-cloud-sandbox'; +import debug from 'debug'; +import { sha256 } from 'js-sha256'; + +import type { + SandboxCommandResult, + SandboxProvider, + SandboxProviderCapabilities, + SandboxProviderKind, + SandboxService, + SandboxServiceOptions, +} from './types'; + +const log = debug('lobe-server:sandbox:service'); + +export class SandboxMiddlewareService implements SandboxService { + readonly capabilities: SandboxProviderCapabilities; + readonly kind: SandboxProviderKind; + + constructor( + private readonly provider: SandboxProvider, + private readonly options: SandboxServiceOptions, + ) { + this.capabilities = provider.capabilities; + this.kind = provider.kind; + } + + callTool(toolName: string, params: Record): Promise { + return this.provider.callTool(toolName, params); + } + + async exportAndUploadFile(path: string, filename: string): Promise { + const { fileService, topicId } = this.options; + + if (!fileService) { + return { + error: { message: 'fileService is required for sandbox file export' }, + filename, + success: false, + }; + } + + log('Exporting file: %s from path: %s, topicId: %s', filename, path, topicId); + + try { + const now = Date.now(); + const today = new Date(now).toISOString().split('T')[0]; + const key = `code-interpreter-exports/${today}/${topicId}/${filename}`; + const upload = await fileService.createPreSignedUpload(key); + + const exported = await this.provider.exportFileToUploadUrl({ + filename, + path, + uploadHeaders: upload.headers, + uploadUrl: upload.url, + }); + + if (!exported.success) { + return { + error: { + message: exported.error?.message || 'Failed to export file from sandbox', + name: exported.error?.name, + }, + filename, + success: false, + }; + } + + const metadata = await fileService.getFileMetadata(key); + const fileSize = metadata.contentLength; + const mimeType = + metadata.contentType || + exported.mimeType || + String(exported.result?.mimeType || '') || + String(exported.result?.mime_type || '') || + 'application/octet-stream'; + const fileHash = sha256(key + now.toString()); + + const { fileId, url } = await fileService.createFileRecord({ + fileHash, + fileType: mimeType, + name: filename, + size: fileSize, + url: key, + }); + + return { + fileId, + filename, + mimeType, + size: fileSize, + success: true, + url, + }; + } catch (error) { + log('Error exporting file: %O', error); + + return { + error: { message: (error as Error).message }, + filename, + success: false, + }; + } + } +} + +export const normalizeSandboxCommandResult = ( + result: SandboxCallToolResult, +): SandboxCommandResult => { + if (!result.success) { + return { + exitCode: 1, + output: '', + stderr: result.error?.message || 'Command execution failed', + success: false, + }; + } + + const raw = result.result || {}; + const rawExitCode = raw.exitCode ?? raw.exit_code; + const exitCode = typeof rawExitCode === 'number' ? rawExitCode : 0; + const output = String(raw.stdout || raw.output || ''); + const stderr = raw.stderr === undefined ? undefined : String(raw.stderr); + const success = typeof raw.success === 'boolean' ? raw.success : exitCode === 0; + + return { + exitCode, + output, + stderr, + success, + }; +}; diff --git a/src/server/services/sandbox/types.ts b/src/server/services/sandbox/types.ts new file mode 100644 index 0000000000..ecf71d8f92 --- /dev/null +++ b/src/server/services/sandbox/types.ts @@ -0,0 +1,70 @@ +import type { + ISandboxService, + SandboxExportFileResult, +} from '@lobechat/builtin-tool-cloud-sandbox'; + +import type { FileService } from '@/server/services/file'; +import type { MarketService } from '@/server/services/market'; + +export type SandboxProviderKind = 'market' | 'onlyboxes'; + +export interface SandboxSessionContext { + topicId: string; + userId: string; +} + +export interface SandboxServiceOptions extends SandboxSessionContext { + fileService?: FileService; + marketService: MarketService; +} + +export interface SandboxProviderCapabilities { + backgroundCommands: boolean; + exportFile: boolean; + files: boolean; + languages: string[]; + persistentSession: boolean; + shell: boolean; + skillScripts: boolean; +} + +export interface SandboxProvider extends Pick { + readonly capabilities: SandboxProviderCapabilities; + + exportFileToUploadUrl: ( + request: SandboxProviderFileExportRequest, + ) => Promise; + + readonly kind: SandboxProviderKind; +} + +export interface SandboxService extends ISandboxService { + readonly capabilities: SandboxProviderCapabilities; + readonly kind: SandboxProviderKind; +} + +export interface SandboxFileExporter { + exportAndUploadFile: (path: string, filename: string) => Promise; +} + +export interface SandboxProviderFileExportRequest { + filename: string; + path: string; + uploadHeaders?: Record; + uploadUrl: string; +} + +export interface SandboxProviderFileExportResult { + error?: { message: string; name?: string }; + mimeType?: string; + result?: Record; + size?: number; + success: boolean; +} + +export interface SandboxCommandResult { + exitCode: number; + output: string; + stderr?: string; + success: boolean; +} diff --git a/src/server/services/skill/importer.test.ts b/src/server/services/skill/importer.test.ts index 3dbd45409f..3f027af14a 100644 --- a/src/server/services/skill/importer.test.ts +++ b/src/server/services/skill/importer.test.ts @@ -61,6 +61,7 @@ vi.stubGlobal('fetch', mockFetch); vi.mock('@/server/services/file/impls', () => ({ createFileServiceModule: vi.fn().mockImplementation(() => ({ createPreSignedUrl: vi.fn().mockResolvedValue('mock-presigned-url'), + createPreSignedUpload: vi.fn().mockResolvedValue({ url: 'mock-presigned-url' }), createPreSignedUrlForPreview: vi.fn().mockResolvedValue('mock-preview-url'), deleteFile: vi.fn().mockResolvedValue(undefined), deleteFiles: vi.fn().mockResolvedValue(undefined), @@ -573,6 +574,7 @@ describe('SkillImporter', () => { const mockUploadBuffer = vi.fn().mockResolvedValue({ key: 'mock-key' }); (createFileServiceModule as any).mockReturnValue({ createPreSignedUrl: vi.fn(), + createPreSignedUpload: vi.fn(), createPreSignedUrlForPreview: vi.fn(), deleteFile: vi.fn(), deleteFiles: vi.fn(), @@ -645,6 +647,7 @@ describe('SkillImporter', () => { const mockUploadBuffer = vi.fn().mockResolvedValue({ key: 'mock-key' }); (createFileServiceModule as any).mockReturnValue({ createPreSignedUrl: vi.fn(), + createPreSignedUpload: vi.fn(), createPreSignedUrlForPreview: vi.fn(), deleteFile: vi.fn(), deleteFiles: vi.fn(), @@ -748,6 +751,7 @@ describe('SkillImporter', () => { const mockUploadBuffer = vi.fn().mockResolvedValue({ key: 'mock-key' }); (createFileServiceModule as any).mockReturnValue({ createPreSignedUrl: vi.fn(), + createPreSignedUpload: vi.fn(), createPreSignedUrlForPreview: vi.fn(), deleteFile: vi.fn(), deleteFiles: vi.fn(), @@ -1013,6 +1017,7 @@ description: A nested skill const { createFileServiceModule } = await import('@/server/services/file/impls'); (createFileServiceModule as any).mockReturnValue({ createPreSignedUrl: vi.fn().mockResolvedValue('mock-presigned-url'), + createPreSignedUpload: vi.fn().mockResolvedValue({ url: 'mock-presigned-url' }), createPreSignedUrlForPreview: vi.fn().mockResolvedValue('mock-preview-url'), deleteFile: vi.fn().mockResolvedValue(undefined), deleteFiles: vi.fn().mockResolvedValue(undefined), diff --git a/src/server/services/toolExecution/serverRuntimes/__tests__/skills.test.ts b/src/server/services/toolExecution/serverRuntimes/__tests__/skills.test.ts new file mode 100644 index 0000000000..2a223b1280 --- /dev/null +++ b/src/server/services/toolExecution/serverRuntimes/__tests__/skills.test.ts @@ -0,0 +1,158 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const sandboxService = { + callTool: vi.fn(), + capabilities: { + backgroundCommands: true, + exportFile: true, + files: true, + languages: ['python'], + persistentSession: true, + shell: true, + skillScripts: true, + }, + exportAndUploadFile: vi.fn(), + kind: 'onlyboxes', + }; + + return { + checkHash: vi.fn(), + createSandboxService: vi.fn(() => sandboxService), + fileService: { + getFullFileUrl: vi.fn(), + }, + findAll: vi.fn(), + findById: vi.fn(), + findByName: vi.fn(), + getAgentSkills: vi.fn(), + getUserSettings: vi.fn(), + marketService: {}, + readResource: vi.fn(), + sandboxService, + }; +}); + +vi.mock('@lobechat/builtin-skills', () => ({ + builtinSkills: [], +})); + +vi.mock('@/database/models/agentSkill', () => ({ + AgentSkillModel: vi.fn(() => ({ + findAll: mocks.findAll, + findById: mocks.findById, + findByName: mocks.findByName, + })), +})); + +vi.mock('@/database/models/file', () => ({ + FileModel: vi.fn(() => ({ + checkHash: mocks.checkHash, + })), +})); + +vi.mock('@/database/models/user', () => ({ + UserModel: vi.fn(() => ({ + getUserSettings: mocks.getUserSettings, + })), +})); + +vi.mock('@/helpers/skillFilters', () => ({ + filterBuiltinSkills: vi.fn((skills: unknown) => skills), +})); + +vi.mock('@/server/services/agentDocuments', () => ({ + AgentDocumentsService: vi.fn(() => ({ + getAgentSkills: mocks.getAgentSkills, + })), +})); + +vi.mock('@/server/services/file', () => ({ + FileService: vi.fn(() => mocks.fileService), +})); + +vi.mock('@/server/services/market', () => ({ + MarketService: vi.fn(() => mocks.marketService), +})); + +vi.mock('@/server/services/sandbox', async () => { + const actual = await vi.importActual('@/server/services/sandbox'); + + return { + ...(actual as Record), + createSandboxService: mocks.createSandboxService, + }; +}); + +vi.mock('@/server/services/skill/resource', () => ({ + SkillResourceService: vi.fn(() => ({ + readResource: mocks.readResource, + })), +})); + +describe('skillsRuntime', () => { + beforeEach(() => { + vi.clearAllMocks(); + + mocks.checkHash.mockResolvedValue({ isExist: true, url: 'skills/user-skill.zip' }); + mocks.fileService.getFullFileUrl.mockResolvedValue('https://files.example.com/user-skill.zip'); + mocks.findAll.mockResolvedValue({ data: [], total: 0 }); + mocks.findById.mockResolvedValue(undefined); + mocks.findByName.mockImplementation(async (name: string) => { + if (name === 'user-skill') { + return { + id: 'user-skill-id', + name: 'user-skill', + zipFileHash: 'zip-hash-1', + }; + } + + return undefined; + }); + mocks.getAgentSkills.mockResolvedValue([]); + mocks.getUserSettings.mockResolvedValue({ market: { accessToken: 'market-token' } }); + mocks.sandboxService.callTool.mockResolvedValue({ + result: { + exitCode: 0, + output: 'ok', + stdout: 'ok', + success: true, + }, + success: true, + }); + }); + + it('executes scripts through the sandbox service and only attaches persisted skill zips', async () => { + const { skillsRuntime } = await import('../skills'); + const runtime = await skillsRuntime.factory({ + serverDB: {} as never, + toolManifestMap: {}, + topicId: 'topic-1', + userId: 'user-1', + }); + + const result = await runtime.execScript({ + activatedSkills: [ + { id: 'user-skill-id', name: 'user-skill' }, + { id: 'builtin-skill-id', name: 'builtin-skill' }, + ], + command: 'python scripts/run.py', + description: 'Run skill script', + }); + + expect(result.success).toBe(true); + expect(mocks.findByName).toHaveBeenCalledWith('user-skill'); + expect(mocks.findByName).toHaveBeenCalledWith('builtin-skill'); + expect(mocks.checkHash).toHaveBeenCalledWith('zip-hash-1'); + expect(mocks.sandboxService.callTool).toHaveBeenCalledWith( + 'execScript', + expect.objectContaining({ + command: 'python scripts/run.py', + description: 'Run skill script', + skillZipUrls: { + 'user-skill': 'https://files.example.com/user-skill.zip', + }, + }), + ); + }); +}); diff --git a/src/server/services/toolExecution/serverRuntimes/cloudSandbox.ts b/src/server/services/toolExecution/serverRuntimes/cloudSandbox.ts index 5b230b4af4..74e46b18b8 100644 --- a/src/server/services/toolExecution/serverRuntimes/cloudSandbox.ts +++ b/src/server/services/toolExecution/serverRuntimes/cloudSandbox.ts @@ -5,7 +5,7 @@ import { import { FileService } from '@/server/services/file'; import { MarketService } from '@/server/services/market'; -import { ServerSandboxService } from '@/server/services/sandbox'; +import { createSandboxService } from '@/server/services/sandbox'; import { type ServerRuntimeRegistration } from './types'; @@ -25,7 +25,7 @@ export const cloudSandboxRuntime: ServerRuntimeRegistration = { const marketService = new MarketService({ userInfo: { userId: context.userId } }); const fileService = new FileService(context.serverDB, context.userId); - const sandboxService = new ServerSandboxService({ + const sandboxService = createSandboxService({ fileService, marketService, topicId: context.topicId, diff --git a/src/server/services/toolExecution/serverRuntimes/skills.ts b/src/server/services/toolExecution/serverRuntimes/skills.ts index d854d13ff4..3ecf07ccac 100644 --- a/src/server/services/toolExecution/serverRuntimes/skills.ts +++ b/src/server/services/toolExecution/serverRuntimes/skills.ts @@ -2,7 +2,11 @@ import { builtinSkills } from '@lobechat/builtin-skills'; import { LocalSystemApiName, LocalSystemIdentifier } from '@lobechat/builtin-tool-local-system'; // Note: only `readFile` is wired through deviceGateway. Directory enumeration is // left to the model via `local-system.listFiles` so we don't double-fetch. -import { type CommandResult, SkillsIdentifier } from '@lobechat/builtin-tool-skills'; +import { + type CommandResult, + type ExecScriptActivatedSkill, + SkillsIdentifier, +} from '@lobechat/builtin-tool-skills'; import { type DeviceFileAccess, type ExportFileResult, @@ -10,18 +14,16 @@ import { SkillsExecutionRuntime, } from '@lobechat/builtin-tool-skills/executionRuntime'; import type { BuiltinSkill, SkillItem, SkillListItem, SkillResourceContent } from '@lobechat/types'; -import type { CodeInterpreterToolName } from '@lobehub/market-sdk'; import debug from 'debug'; -import { sha256 } from 'js-sha256'; import { AgentSkillModel } from '@/database/models/agentSkill'; import { FileModel } from '@/database/models/file'; import { UserModel } from '@/database/models/user'; import { filterBuiltinSkills } from '@/helpers/skillFilters'; -import { FileS3 } from '@/server/modules/S3'; import { AgentDocumentsService } from '@/server/services/agentDocuments'; import { FileService } from '@/server/services/file'; import { MarketService } from '@/server/services/market'; +import { createSandboxService, normalizeSandboxCommandResult } from '@/server/services/sandbox'; import { SkillResourceService } from '@/server/services/skill/resource'; import { preprocessLhCommand } from '@/server/services/toolExecution/preprocessLhCommand'; @@ -30,6 +32,12 @@ import { type ServerRuntimeRegistration } from './types'; const log = debug('lobe-server:skills-runtime'); +interface UserSettingsWithMarketToken { + market?: { + accessToken?: string; + }; +} + class SkillServerRuntimeService implements SkillRuntimeService { private resourceService: SkillResourceService; private skillModel: AgentSkillModel; @@ -88,12 +96,13 @@ class SkillServerRuntimeService implements SkillRuntimeService { } try { - const market = this.marketService.market; - const response = await market.plugins.runBuildInTool( - 'runCommand' as any, - { command: lhResult.command }, - { topicId: this.topicId, userId: this.userId }, - ); + const sandboxService = createSandboxService({ + fileService: this.fileService, + marketService: this.marketService, + topicId: this.topicId, + userId: this.userId, + }); + const response = await sandboxService.callTool('runCommand', { command: lhResult.command }); log('runCommand response: %O', response); @@ -106,14 +115,7 @@ class SkillServerRuntimeService implements SkillRuntimeService { }; } - const result = response.data?.result || {}; - - return { - exitCode: result.exitCode ?? (response.success ? 0 : 1), - output: result.stdout || result.output || '', - stderr: result.stderr || '', - success: response.success && (result.exitCode === 0 || result.exitCode === undefined), - }; + return normalizeSandboxCommandResult(response); } catch (error) { log('Error running command: %O', error); return { @@ -128,69 +130,61 @@ class SkillServerRuntimeService implements SkillRuntimeService { execScript = async ( command: string, options: { - config?: { description?: string; id?: string; name?: string }; + activatedSkills?: ExecScriptActivatedSkill[]; description: string; - runInClient?: boolean; }, ): Promise => { - const { config, description } = options; + const { activatedSkills, description } = options; if (!this.topicId) { throw new Error('topicId is required for execScript'); } try { - // Look up skill zipUrl if config is provided (same logic as market.ts) - const enhancedParams: any = { + const enhancedParams: Record = { + activatedSkills, command, - config, description, }; - if (config?.name) { - const skill = await this.skillModel.findByName(config.name); + if (activatedSkills?.length) { + const skillZipUrls: Record = {}; - // If skill not found, return error with available skills - if (!skill) { - const allSkills = await this.skillModel.findAll(); - const availableSkills = allSkills.data.map((s) => s.name).join(', '); + for (const activatedSkill of activatedSkills) { + if (!activatedSkill.name) continue; - const errorMessage = availableSkills - ? `Skill "${config.name}" not found. Available skills: ${availableSkills}` - : `Skill "${config.name}" not found. No skills available. Please import a skill first.`; + const skill = await this.skillModel.findByName(activatedSkill.name); - log('Skill not found: %s. Available skills: %s', config.name, availableSkills); + if (!skill) { + log('No persisted skill bundle found for activated skill: %s', activatedSkill.name); + continue; + } - return { - exitCode: 1, - output: '', - stderr: errorMessage, - success: false, - }; + if (!skill.zipFileHash) continue; + + const fileInfo = await this.fileModel.checkHash(skill.zipFileHash); + if (!fileInfo.isExist || !fileInfo.url) continue; + + const fullUrl = await this.fileService.getFullFileUrl(fileInfo.url); + if (fullUrl) { + skillZipUrls[skill.name] = fullUrl; + log('Resolved zipUrl for skill %s', skill.name); + } } - if (skill.zipFileHash) { - // Get S3 key from globalFiles - const fileInfo = await this.fileModel.checkHash(skill.zipFileHash); - - if (fileInfo.isExist && fileInfo.url) { - // Convert S3 key to full URL - const fullUrl = await this.fileService.getFullFileUrl(fileInfo.url); - if (fullUrl) { - enhancedParams.zipUrl = fullUrl; - log('Added zipUrl to execScript params for skill %s: %s', skill.name, fullUrl); - } - } + if (Object.keys(skillZipUrls).length > 0) { + enhancedParams.skillZipUrls = skillZipUrls; + log('Added skillZipUrls to execScript params: %O', Object.keys(skillZipUrls)); } } - // Call market-sdk's runBuildInTool - const market = this.marketService.market; - const response = await market.plugins.runBuildInTool( - 'execScript' as CodeInterpreterToolName, - enhancedParams, - { topicId: this.topicId, userId: this.userId }, - ); + const sandboxService = createSandboxService({ + fileService: this.fileService, + marketService: this.marketService, + topicId: this.topicId, + userId: this.userId, + }); + const response = await sandboxService.callTool('execScript', enhancedParams); log('execScript response: %O', response); @@ -203,14 +197,7 @@ class SkillServerRuntimeService implements SkillRuntimeService { }; } - const result = response.data?.result || {}; - - return { - exitCode: result.exitCode ?? (response.success ? 0 : 1), - output: result.stdout || result.output || '', - stderr: result.stderr || '', - success: response.success && (result.exitCode === 0 || result.exitCode === undefined), - }; + return normalizeSandboxCommandResult(response); } catch (error) { log('Error executing script: %O', error); return { @@ -228,68 +215,21 @@ class SkillServerRuntimeService implements SkillRuntimeService { } try { - const s3 = new FileS3(); - - // Use date-based sharding (same as market.ts) - const today = new Date().toISOString().split('T')[0]; - const key = `code-interpreter-exports/${today}/${this.topicId}/${filename}`; - - // Step 1: Generate pre-signed upload URL - const uploadUrl = await s3.createPreSignedUrl(key); - log('Generated upload URL for key: %s', key); - - // Step 2: Call sandbox's exportFile tool with the upload URL - const market = this.marketService.market; - const response = await market.plugins.runBuildInTool( - 'exportFile' as CodeInterpreterToolName, - { path, uploadUrl }, - { topicId: this.topicId, userId: this.userId }, - ); - - log('Sandbox exportFile response: %O', response); - - if (!response.success) { - return { - filename, - success: false, - }; - } - - const result = response.data?.result; - const uploadSuccess = result?.success !== false; - - if (!uploadSuccess) { - return { - filename, - success: false, - }; - } - - // Step 3: Get file metadata from S3 - const metadata = await s3.getFileMetadata(key); - const fileSize = metadata.contentLength; - const mimeType = metadata.contentType || result?.mimeType || 'application/octet-stream'; - - // Step 4: Create persistent file record - const fileHash = sha256(key + Date.now().toString()); - - const { fileId, url } = await this.fileService.createFileRecord({ - fileHash, - fileType: mimeType, - name: filename, - size: fileSize, - url: key, // Store S3 key + const sandboxService = createSandboxService({ + fileService: this.fileService, + marketService: this.marketService, + topicId: this.topicId, + userId: this.userId, }); - - log('Created file record: fileId=%s, url=%s', fileId, url); + const result = await sandboxService.exportAndUploadFile(path, filename); return { - fileId, - filename, - mimeType, - size: fileSize, - success: true, - url, // This is the permanent /f:id URL + fileId: result.fileId, + filename: result.filename, + mimeType: result.mimeType, + size: result.size, + success: result.success, + url: result.url, }; } catch (error) { log('Error exporting file: %O', error); @@ -319,7 +259,8 @@ export const skillsRuntime: ServerRuntimeRegistration = { try { const userModel = new UserModel(context.serverDB, context.userId); const userSettings = await userModel.getUserSettings(); - marketAccessToken = (userSettings?.market as any)?.accessToken; + marketAccessToken = (userSettings as UserSettingsWithMarketToken | undefined)?.market + ?.accessToken; log( 'Fetched market accessToken for user %s: %s', context.userId,