Compare commits

...

1 Commits

Author SHA1 Message Date
Arvin Xu a6c2a57f54 feat(agent-runtime): support client-side tool dispatch via device gateway
- Add clientRuntime field to AgentToolsEngine to support tool dispatch from client
- Add clientRuntime type to BuiltinToolManifest for client-executable tools
- Update AgentToolsEngine to route clientRuntime tools through gateway sendToolExecute
- Add deviceToolPipeline tests for client-side tool execution flow
- Update aiAgent router/service to pass gateway context for client dispatch
- Update gateway store action to handle tool execution requests from server
2026-05-27 12:33:52 +08:00
10 changed files with 304 additions and 48 deletions
@@ -58,6 +58,13 @@ export interface ExecAgentParams {
appContext?: ExecAgentAppContext;
/** Whether to auto-start execution after creating operation (default: true) */
autoStart?: boolean;
/**
* Runtime of the client initiating this request. Used by the server to
* enable `executor: 'client'` tools (e.g. local-system) when the caller
* is a desktop Electron client that will receive `tool_execute` events
* over the same Agent Gateway WebSocket.
*/
clientRuntime?: 'desktop' | 'web';
/** Explicit device ID to bind to the topic and activate for this run */
deviceId?: string;
/** Optional existing message IDs to include in context */
+5 -4
View File
@@ -191,12 +191,13 @@ export interface BuiltinToolManifest {
/**
* Supported execution environments for this tool.
* - `'client'`: executed in-process by an embedded Electron runtime that
* hosts both the server and the executor. Used only by standalone
* builds without a device-gateway. Deployments with DEVICE_GATEWAY
* route the same tools through the device-gateway proxy instead.
* - `'client'`: dispatched to the client via Agent Gateway WebSocket
* (requires Electron / desktop runtime). For tools that depend on
* local resources (filesystem, EditorRuntime, stdio MCP, etc.).
* - `'server'`: executed server-side by ToolExecutionService.
*
* When both are present, the server picks based on `clientRuntime`:
* desktop callers get `'client'` dispatch; web callers get `'server'`.
* When omitted, defaults to server-only execution.
*/
executors?: ('client' | 'server')[];
@@ -508,8 +508,8 @@ describe('createServerAgentToolsEngine', () => {
describe('RemoteDevice tool enable rules', () => {
// Same pattern as LocalSystem above: `canUseDevice: true` is set so the
// assertions exercise the engine-internal gates (gatewayConfigured,
// autoActivated). The `canUseDevice gate` block below covers the
// policy-level gating.
// autoActivated, hasClientExecutor). The `canUseDevice gate` block
// below covers the policy-level gating.
it('should enable RemoteDevice when gateway configured and no device auto-activated', () => {
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
@@ -662,15 +662,133 @@ describe('createServerAgentToolsEngine', () => {
});
});
describe('clientRuntime === "desktop" (Phase 6.4)', () => {
it('enables LocalSystem when caller is desktop, regardless of device-proxy config', () => {
// The Agent Gateway WS used to push `tool_execute` is orthogonal to
// the legacy device-proxy. A desktop Electron caller is already the
// execution target — no device-proxy prerequisite required.
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
agentConfig: { plugins: [LocalSystemManifest.identifier] },
canUseDevice: true,
clientRuntime: 'desktop',
model: 'gpt-4',
provider: 'openai',
});
const result = engine.generateToolsDetailed({
toolIds: [LocalSystemManifest.identifier],
model: 'gpt-4',
provider: 'openai',
});
expect(result.enabledToolIds).toContain(LocalSystemManifest.identifier);
});
it('respects agent-level runtimeMode opt-out for desktop callers', () => {
// User has configured the agent to NOT use local runtime on desktop.
// Even though the caller is a desktop client, local-system stays off.
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
agentConfig: {
chatConfig: {
runtimeEnv: { runtimeMode: { desktop: 'none' } },
},
plugins: [LocalSystemManifest.identifier],
},
canUseDevice: true,
clientRuntime: 'desktop',
model: 'gpt-4',
provider: 'openai',
});
const result = engine.generateToolsDetailed({
toolIds: [LocalSystemManifest.identifier],
model: 'gpt-4',
provider: 'openai',
});
expect(result.enabledToolIds).not.toContain(LocalSystemManifest.identifier);
});
it('can suppress only LocalSystem while preserving the rest of tool discovery', () => {
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
agentConfig: { plugins: [LocalSystemManifest.identifier, WebBrowsingManifest.identifier] },
canUseDevice: true,
clientRuntime: 'desktop',
disableLocalSystem: true,
model: 'gpt-4',
provider: 'openai',
});
const result = engine.generateToolsDetailed({
model: 'gpt-4',
provider: 'openai',
toolIds: [LocalSystemManifest.identifier, WebBrowsingManifest.identifier],
});
expect(result.enabledToolIds).not.toContain(LocalSystemManifest.identifier);
expect(result.enabledToolIds).toContain(WebBrowsingManifest.identifier);
});
it('does not enable LocalSystem for web callers even when gateway is configured', () => {
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
agentConfig: { plugins: [LocalSystemManifest.identifier] },
canUseDevice: true,
clientRuntime: 'web',
deviceContext: { gatewayConfigured: true },
model: 'gpt-4',
provider: 'openai',
});
const result = engine.generateToolsDetailed({
toolIds: [LocalSystemManifest.identifier],
model: 'gpt-4',
provider: 'openai',
});
expect(result.enabledToolIds).not.toContain(LocalSystemManifest.identifier);
});
it('suppresses RemoteDevice when caller is a desktop client', () => {
// Even when device-proxy is configured, a desktop caller has local IPC
// so the proxy is redundant. Otherwise the LLM might pick RemoteDevice
// first (via `listOnlineDevices` / `activateDevice`) and route tool calls
// to a *different* registered device instead of back to the caller.
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
agentConfig: {
plugins: [LocalSystemManifest.identifier, RemoteDeviceManifest.identifier],
},
canUseDevice: true,
clientRuntime: 'desktop',
deviceContext: { gatewayConfigured: true },
model: 'gpt-4',
provider: 'openai',
});
const result = engine.generateToolsDetailed({
toolIds: [LocalSystemManifest.identifier, RemoteDeviceManifest.identifier],
model: 'gpt-4',
provider: 'openai',
});
expect(result.enabledToolIds).toContain(LocalSystemManifest.identifier);
expect(result.enabledToolIds).not.toContain(RemoteDeviceManifest.identifier);
});
});
describe('canUseDevice gate (device access policy)', () => {
it('drops LocalSystem when canUseDevice is false even with a healthy device-gateway', () => {
// External bot sender impersonating an owner must not pull local-system
// into the tool list even when every other gate would normally pass.
it('drops LocalSystem when canUseDevice is false even on a desktop caller', () => {
// External bot sender impersonating a desktop session must not get
// local-system back through Phase 6.4 dispatch.
const context = createMockContext();
const engine = createServerAgentToolsEngine(context, {
agentConfig: { plugins: [LocalSystemManifest.identifier] },
canUseDevice: false,
deviceContext: { gatewayConfigured: true, deviceOnline: true, autoActivated: true },
clientRuntime: 'desktop',
model: 'gpt-4',
provider: 'openai',
});
@@ -711,6 +829,7 @@ describe('createServerAgentToolsEngine', () => {
agentConfig: {
plugins: [LocalSystemManifest.identifier, RemoteDeviceManifest.identifier],
},
clientRuntime: 'desktop',
deviceContext: { gatewayConfigured: true, deviceOnline: true, autoActivated: true },
model: 'gpt-4',
provider: 'openai',
@@ -126,6 +126,7 @@ export const createServerAgentToolsEngine = (
additionalManifests,
agentConfig,
canUseDevice = false,
clientRuntime,
deviceContext,
disableLocalSystem = false,
globalMemoryEnabled = false,
@@ -136,17 +137,28 @@ export const createServerAgentToolsEngine = (
provider,
} = params;
// Tools that need a user-side execution target (local-system, stdio MCP)
// run on a device registered with the device-gateway. Desktop, CLI, and
// bot/IM callers all converge on this single path; the previous Phase 6.4
// `clientRuntime === 'desktop'` short-circuit (Agent Gateway WS dispatch
// back to the caller) is removed — see LOBE-9378.
// ─── Tool-dispatch capability flags ───
//
// Two orthogonal signals control whether client-side tools can run.
//
// 1. `hasClientExecutor` — the caller itself is an Electron desktop
// client and can receive `tool_execute` events over the Agent
// Gateway WebSocket (Phase 6.4).
// 2. `hasDeviceProxy` — the server has a device-proxy configured that
// can tunnel commands to a *separately registered* desktop device
// (legacy Remote Device flow).
//
// Either, both, or neither can be true independently.
const hasClientExecutor = clientRuntime === 'desktop';
const hasDeviceProxy = !!deviceContext?.gatewayConfigured;
// Platform key is used only to look up the user's per-platform
// `runtimeMode` preference. A server configured with a device-gateway is
// serving desktop-class users; otherwise the caller is treated as web.
const platform: RuntimePlatform = hasDeviceProxy ? 'desktop' : 'web';
// ─── Platform / runtime mode ───
//
// `platform` is a property of the caller, not of the server. Prefer the
// explicit `clientRuntime` signal; fall back to treating a server with
// a configured device-proxy as desktop for callers that don't yet send
// `clientRuntime` (backwards compat).
const platform: RuntimePlatform = clientRuntime ?? (hasDeviceProxy ? 'desktop' : 'web');
// User-configured runtime mode for the current platform, with a
// platform-appropriate default when unset.
@@ -159,13 +171,14 @@ export const createServerAgentToolsEngine = (
const isChatMode = agentConfig.chatConfig?.enableAgentMode === false;
log(
'Creating agent tools engine model=%s provider=%s searchMode=%s platform=%s runtimeMode=%s additionalManifests=%d hasDeviceProxy=%s canUseDevice=%s isChatMode=%s',
'Creating agent tools engine model=%s provider=%s searchMode=%s platform=%s runtimeMode=%s additionalManifests=%d hasClientExecutor=%s hasDeviceProxy=%s canUseDevice=%s isChatMode=%s',
model,
provider,
searchMode,
platform,
runtimeMode,
additionalManifests?.length ?? 0,
hasClientExecutor,
hasDeviceProxy,
canUseDevice,
isChatMode,
@@ -193,20 +206,24 @@ export const createServerAgentToolsEngine = (
// Local-system: gated by `canUseDevice` (resolveDeviceAccessPolicy)
// first — keeps external bot senders out before runtime checks even
// run. Then user must have opted into local runtime on this platform
// (`runtimeMode === 'local'`) AND have an online, auto-activated
// device registered with the device-gateway.
// (`runtimeMode === 'local'`), AND one execution channel must exist:
// - `hasClientExecutor` — Phase 6.4 dispatch over the Agent Gateway
// WS that this request is already riding on; no extra server-side
// prerequisite needed;
// - legacy device-proxy with an online & auto-activated device.
[LocalSystemManifest.identifier]:
canUseDevice &&
!disableLocalSystem &&
runtimeMode === 'local' &&
hasDeviceProxy &&
!!deviceContext?.deviceOnline &&
!!deviceContext?.autoActivated,
(hasClientExecutor ||
(hasDeviceProxy && !!deviceContext?.deviceOnline && !!deviceContext?.autoActivated)),
[MemoryManifest.identifier]: globalMemoryEnabled,
// Only auto-enable in bot conversations; otherwise let user's plugin selection take effect
...(isBotConversation && { [MessageManifest.identifier]: true }),
// Remote-device proxy: shown only when the server has a proxy but
// no specific device is auto-activated yet (user must pick).
// no specific device is auto-activated yet (user must pick). When
// the caller itself can execute `executor: 'client'` tools, the
// proxy is redundant — local-system goes directly to the caller.
//
// `canUseDevice` is the first short-circuit: external bot senders
// (and unconfigured bot owners) never reach the proxy, both because
@@ -214,7 +231,7 @@ export const createServerAgentToolsEngine = (
// systemRole would otherwise leak the device list into the LLM
// context — see the gated injection in `aiAgent.execAgent`.
[RemoteDeviceManifest.identifier]:
canUseDevice && hasDeviceProxy && !deviceContext?.autoActivated,
canUseDevice && hasDeviceProxy && !deviceContext?.autoActivated && !hasClientExecutor,
[AgentDocumentsManifest.identifier]: hasAgentDocuments,
[WebBrowsingManifest.identifier]: isSearchEnabled,
};
@@ -75,6 +75,13 @@ export interface ServerCreateAgentToolsEngineParams {
* Defaults to `false` (fail-closed) when the caller forgets to plumb it.
*/
canUseDevice?: boolean;
/**
* Runtime of the client initiating this request. When `'desktop'`, the
* caller itself is an Electron client connected via the Agent Gateway WS,
* so tools with `executor: 'client'` (e.g. local-system, stdio MCP) can be
* dispatched back to it via `tool_execute` — no remote-device proxy needed.
*/
clientRuntime?: 'desktop' | 'web';
/** Device gateway context for remote tool calling */
deviceContext?: {
/** When true, a device has been auto-activated — Remote Device tool is unnecessary */
+8
View File
@@ -146,6 +146,12 @@ const ExecAgentSchema = z
.optional(),
/** Whether to auto-start execution after creating operation */
autoStart: z.boolean().optional().default(true),
/**
* Runtime of the client initiating this request.
* 'desktop' enables `executor: 'client'` tools (local-system, stdio MCP)
* to be dispatched over the Agent Gateway WS.
*/
clientRuntime: z.enum(['desktop', 'web']).optional(),
/** Explicit device ID to bind to the topic and activate for this run */
deviceId: z.string().optional(),
/** Optional existing message IDs to include in context */
@@ -627,6 +633,7 @@ export const aiAgentRouter = router({
prompt,
appContext,
autoStart = true,
clientRuntime,
deviceId,
existingMessageIds = [],
fileIds,
@@ -644,6 +651,7 @@ export const aiAgentRouter = router({
agentId,
appContext,
autoStart,
clientRuntime,
deviceId,
existingMessageIds,
fileIds,
@@ -229,6 +229,44 @@ describe('AiAgentService.execAgent - device tool pipeline ()', () => {
});
});
describe('clientRuntime forwarded to createServerAgentToolsEngine', () => {
it('forwards clientRuntime="desktop" so the engine enables local-system for Electron callers', async () => {
mockGetAgentConfig.mockResolvedValue(createBaseAgentConfig());
await service.execAgent({
agentId: 'agent-1',
clientRuntime: 'desktop',
prompt: 'Hello',
});
expect(mockCreateServerAgentToolsEngine).toHaveBeenCalledTimes(1);
const params = mockCreateServerAgentToolsEngine.mock.calls[0][1];
expect(params.clientRuntime).toBe('desktop');
});
it('forwards clientRuntime="web" verbatim', async () => {
mockGetAgentConfig.mockResolvedValue(createBaseAgentConfig());
await service.execAgent({
agentId: 'agent-1',
clientRuntime: 'web',
prompt: 'Hello',
});
const params = mockCreateServerAgentToolsEngine.mock.calls[0][1];
expect(params.clientRuntime).toBe('web');
});
it('omits clientRuntime when the caller does not specify one', async () => {
mockGetAgentConfig.mockResolvedValue(createBaseAgentConfig());
await service.execAgent({ agentId: 'agent-1', prompt: 'Hello' });
const params = mockCreateServerAgentToolsEngine.mock.calls[0][1];
expect(params.clientRuntime).toBeUndefined();
});
});
describe('RemoteDevice systemRole override', () => {
it('should override RemoteDevice systemRole with dynamic prompt when enabled by ToolsEngine', async () => {
const { deviceProxy } = await import('@/server/services/toolExecution/deviceProxy');
@@ -354,12 +392,14 @@ describe('AiAgentService.execAgent - device tool pipeline ()', () => {
});
});
describe('DEVICE_GATEWAY routing for local-system and stdio MCP', () => {
it('keeps executor unset for local-system when DEVICE_GATEWAY is configured', async () => {
// Desktop, web, and IM callers all share this path: tools route via the
// Remote Device proxy to the device registered with the gateway, never
// back to the caller. (LOBE-9378: the Phase 6.4 clientRuntime=desktop
// short-circuit that bypassed this gate was removed.)
describe('clientRuntime="desktop" bypasses the DEVICE_GATEWAY gate (Phase 6.4)', () => {
it('marks local-system as client when caller is desktop, even with DEVICE_GATEWAY configured', async () => {
// On cloud canary, DEVICE_GATEWAY is configured AND a remote Linux VM
// may be registered. Before this fix, `!gatewayConfigured` was false, so
// local-system was never stamped `executor='client'` — and dispatch fell
// through to the Remote Device proxy (which then tried to read the file
// on the wrong host). When clientRuntime='desktop', the caller itself is
// the execution target and wins.
const { deviceProxy } = await import('@/server/services/toolExecution/deviceProxy');
vi.spyOn(deviceProxy, 'isConfigured', 'get').mockReturnValue(true);
mockQueryDeviceList.mockResolvedValue([
@@ -371,13 +411,17 @@ describe('AiAgentService.execAgent - device tool pipeline ()', () => {
);
mockGetAgentConfig.mockResolvedValue(createBaseAgentConfig());
await service.execAgent({ agentId: 'agent-1', prompt: 'Hello' });
await service.execAgent({
agentId: 'agent-1',
clientRuntime: 'desktop',
prompt: 'Hello',
});
const executorMap = mockCreateOperation.mock.calls[0][0].toolSet.executorMap;
expect(executorMap[LocalSystemManifest.identifier]).toBeUndefined();
expect(executorMap[LocalSystemManifest.identifier]).toBe('client');
});
it('keeps executor unset for stdio MCP when DEVICE_GATEWAY is configured', async () => {
it('marks stdio MCP as client when caller is desktop, even with DEVICE_GATEWAY configured', async () => {
const stdioPlugin = {
customParams: { mcp: { type: 'stdio' } },
identifier: 'my-stdio-mcp',
@@ -398,10 +442,38 @@ describe('AiAgentService.execAgent - device tool pipeline ()', () => {
mockGetEnabledPluginManifests.mockReturnValue(new Map([['my-stdio-mcp', stdioManifest]]));
mockGetAgentConfig.mockResolvedValue(createBaseAgentConfig({ plugins: ['my-stdio-mcp'] }));
await service.execAgent({ agentId: 'agent-1', prompt: 'Hello' });
await service.execAgent({
agentId: 'agent-1',
clientRuntime: 'desktop',
prompt: 'Hello',
});
const executorMap = mockCreateOperation.mock.calls[0][0].toolSet.executorMap;
expect(executorMap['my-stdio-mcp']).toBeUndefined();
expect(executorMap['my-stdio-mcp']).toBe('client');
});
it('keeps legacy routing for web callers with DEVICE_GATEWAY configured', async () => {
// Web client + DEVICE_GATEWAY configured → tools still route through
// Remote Device proxy; executor stays unset (legacy behaviour).
const { deviceProxy } = await import('@/server/services/toolExecution/deviceProxy');
vi.spyOn(deviceProxy, 'isConfigured', 'get').mockReturnValue(true);
mockQueryDeviceList.mockResolvedValue([
{ deviceId: 'dev-1', deviceName: 'Remote VM', platform: 'linux' },
]);
mockGetEnabledPluginManifests.mockReturnValue(
new Map([[LocalSystemManifest.identifier, LocalSystemManifest]]),
);
mockGetAgentConfig.mockResolvedValue(createBaseAgentConfig());
await service.execAgent({
agentId: 'agent-1',
clientRuntime: 'web',
prompt: 'Hello',
});
const executorMap = mockCreateOperation.mock.calls[0][0].toolSet.executorMap;
expect(executorMap[LocalSystemManifest.identifier]).toBeUndefined();
});
});
+25 -10
View File
@@ -306,6 +306,7 @@ export class AiAgentService {
appContext,
autoStart = true,
botContext,
clientRuntime,
deviceId: requestedDeviceId,
botPlatformContext,
discordContext,
@@ -1210,6 +1211,7 @@ export class AiAgentService {
plugins: agentPlugins,
},
canUseDevice,
clientRuntime,
deviceContext: gatewayConfigured
? {
autoActivated: activeDeviceId ? true : undefined,
@@ -1306,22 +1308,35 @@ export class AiAgentService {
toolSourceMap[manifest.identifier] = 'klavis';
}
// Mark tools that must run on the user's machine (local-system, stdio
// MCP) for direct client dispatch only in the standalone deployment
// where no DEVICE_GATEWAY is configured. In that mode the legacy
// Remote Device proxy isn't available and the embedded Electron runs
// both the server and the executor, so tools route in-process.
// Mark tools that must run on the client (desktop Electron) because they
// require local IPC / subprocess capabilities:
// - local-system builtin: Electron IPC for file + command execution
// - stdio MCP plugins: subprocess lives on the user's machine
//
// With a device-gateway configured, every caller (desktop UI, web,
// IM/bot) converges on the device-gateway path: tool calls tunnel to
// a registered device's WS connection. `executor` stays unset so the
// RemoteDevice proxy resolves the route.
if (!gatewayConfigured) {
// Two triggers, in priority order:
// (a) `clientRuntime === 'desktop'` — the caller itself is an Electron
// client on the Agent Gateway WS and is ready to receive
// `tool_execute`. This is the Phase 6.4 path and is authoritative
// regardless of whether DEVICE_GATEWAY (the legacy device-proxy) is
// also configured.
// (b) `!gatewayConfigured` — no DEVICE_GATEWAY configured on the server,
// so legacy Remote Device proxy isn't an option and any client
// tooling falls through to the Gateway WS (standalone Electron).
//
// When DEVICE_GATEWAY is configured AND the caller is a web client, we
// leave executor unset so tools route via RemoteDevice proxy.
const shouldDispatchToClient = clientRuntime === 'desktop' || !gatewayConfigured;
if (shouldDispatchToClient) {
// Tools that declare `executors` including `'client'` in their
// manifest are dispatched to the client when a desktop caller is
// connected. `toolManifestMap` is a superset of `manifestMap`
// (includes both enabled plugins and discoverable builtins).
for (const id of Object.keys(toolManifestMap)) {
if (toolManifestMap[id]?.executors?.includes('client')) {
toolExecutorMap[id] = 'client';
}
}
// Stdio MCP plugins: subprocess lives on the user's machine
for (const plugin of installedPlugins) {
if (plugin.customParams?.mcp?.type === 'stdio' && manifestMap.has(plugin.identifier)) {
toolExecutorMap[plugin.identifier] = 'client';
+6
View File
@@ -28,6 +28,12 @@ export interface ExecAgentTaskParams {
agentId?: string;
appContext?: ExecAgentAppContext;
autoStart?: boolean;
/**
* Runtime of the client initiating this request. When 'desktop', server
* enables `executor: 'client'` tools (local-system, stdio MCP) and
* dispatches them over the Agent Gateway WS back to this client.
*/
clientRuntime?: 'desktop' | 'web';
deviceId?: string;
existingMessageIds?: string[];
/** File IDs of already-uploaded attachments to attach to the new user message */
@@ -368,6 +368,10 @@ export class GatewayActionImpl {
threadId: context.threadId,
topicId: context.topicId,
},
// Tell the server this caller is a desktop Electron client so it can
// enable `executor: 'client'` tools (local-system, stdio MCP) and
// dispatch them back over the Agent Gateway WS.
clientRuntime: isDesktop ? 'desktop' : 'web',
fileIds,
parentMessageId,
projectSkills,