The get_screenshot MCP tool serialized its PNG through OkJson, which wraps the payload in a text content block. Vision-capable MCP clients (Copilot, Claude Code) therefore received a large base64 string as text instead of an image, making the screenshot unusable for visual reasoning (openpencil issue #204). - Add ToolOutcome::OkImageContent carrying base64 + mime_type plus an optional metadata JSON string - Thread an optional ImageContent through ToolResponse::Ok and emit it as an MCP {"type":"image","data":...,"mimeType":...} content block in tool_response_to_json before any text block - get_screenshot now returns OkImageContent (image/png) while retaining image_base64 in its text metadata for the in-app chat-agent path - Update the get_screenshot schema description to state the use case (PNG image for visual verification) without the base64 implementation detail
126 lines
4.6 KiB
Rust
126 lines
4.6 KiB
Rust
//! Manual JSON-RPC wire serializer for `ToolResponse` + internal
|
|
//! helpers. Hand-rolled so shell-core stays serde-free (wasm32
|
|
//! bundle size). Carved off `mcp.rs` to stay under the 800-line cap.
|
|
|
|
use std::collections::BTreeMap;
|
|
|
|
use super::{RequestId, ToolErrorCode, ToolResponse};
|
|
|
|
/// JSON-RPC wire serialiser for `ToolResponse`. Produces the
|
|
/// standard `{"jsonrpc": "2.0", "id": ..., "result": ...}` /
|
|
/// `{"jsonrpc": "2.0", "id": ..., "error": {"code": ..., "message":
|
|
/// ...}}` shape any MCP client expects.
|
|
pub fn response_to_json(r: &ToolResponse) -> String {
|
|
let (id_repr, body) = match r {
|
|
ToolResponse::Ok {
|
|
id, result, json, ..
|
|
} => (
|
|
id_to_json(id),
|
|
match json {
|
|
// Nested-JSON read result: embed verbatim as the wire result.
|
|
Some(raw) => format!(r#""result":{raw}"#),
|
|
None => format!(r#""result":{}"#, btree_to_json(result)),
|
|
},
|
|
),
|
|
ToolResponse::Err { id, code, message } => (
|
|
id_to_json(id),
|
|
format!(
|
|
r#""error":{{"code":{},"message":{}}}"#,
|
|
error_code_to_int(*code),
|
|
json_escape(message),
|
|
),
|
|
),
|
|
};
|
|
format!(r#"{{"jsonrpc":"2.0","id":{},{}}}"#, id_repr, body)
|
|
}
|
|
|
|
/// JSON-RPC serializer for a *tool* result in the MCP-spec `tools/call`
|
|
/// shape, matching TS `pen-mcp` (`server.ts`): the tool's data rides inside
|
|
/// `result.content` as a single `text` block (the flat result map serialized
|
|
/// to a JSON string), and a tool-level failure becomes `isError:true` in the
|
|
/// result — NOT a JSON-RPC `error` (those are reserved for transport/parse
|
|
/// failures, still emitted via [`response_to_json`]). External MCP clients
|
|
/// (Claude Code / Codex) require this envelope.
|
|
///
|
|
/// When `image` is `Some`, an MCP `ImageContent` block is emitted BEFORE
|
|
/// any text block so vision-capable MCP clients receive the image as
|
|
/// multimodal content instead of a base64 string buried in JSON text.
|
|
pub fn tool_response_to_json(r: &ToolResponse) -> String {
|
|
let (id_repr, body) = match r {
|
|
ToolResponse::Ok {
|
|
id,
|
|
result,
|
|
json,
|
|
image,
|
|
..
|
|
} => {
|
|
let mut content_blocks: Vec<String> = Vec::new();
|
|
// Image block first (when present) so vision models see it.
|
|
if let Some(img) = image {
|
|
content_blocks.push(format!(
|
|
r#"{{"type":"image","data":{},"mimeType":{}}}"#,
|
|
json_escape(&img.data),
|
|
json_escape(&img.mime_type),
|
|
));
|
|
}
|
|
// Text block: nested JSON or flat string-map.
|
|
let text = match json {
|
|
Some(raw) => json_escape(raw),
|
|
None => json_escape(&btree_to_json(result)),
|
|
};
|
|
content_blocks.push(format!(r#"{{"type":"text","text":{text}}}"#));
|
|
(
|
|
id_to_json(id),
|
|
format!(r#""result":{{"content":[{}]}}"#, content_blocks.join(",")),
|
|
)
|
|
}
|
|
ToolResponse::Err { id, message, .. } => (
|
|
id_to_json(id),
|
|
format!(
|
|
r#""result":{{"content":[{{"type":"text","text":{}}}],"isError":true}}"#,
|
|
json_escape(&format!("Error: {message}")),
|
|
),
|
|
),
|
|
};
|
|
format!(r#"{{"jsonrpc":"2.0","id":{},{}}}"#, id_repr, body)
|
|
}
|
|
|
|
pub(super) fn id_to_json(id: &RequestId) -> String {
|
|
match id {
|
|
RequestId::Str(s) => json_escape(s),
|
|
RequestId::Num(n) => n.to_string(),
|
|
}
|
|
}
|
|
|
|
pub(super) fn error_code_to_int(code: ToolErrorCode) -> i32 {
|
|
// JSON-RPC reserves -32600..-32603 for transport-level errors;
|
|
// tool errors live in the application range (-32000..-32099).
|
|
match code {
|
|
ToolErrorCode::MissingArgument => -32_001,
|
|
ToolErrorCode::InvalidArgument => -32_602,
|
|
ToolErrorCode::ToolFailed => -32_002,
|
|
ToolErrorCode::UnknownTool => -32_601,
|
|
ToolErrorCode::Internal => -32_603,
|
|
}
|
|
}
|
|
|
|
pub(super) fn btree_to_json(m: &BTreeMap<String, String>) -> String {
|
|
let mut out = String::from("{");
|
|
let mut first = true;
|
|
for (k, v) in m {
|
|
if !first {
|
|
out.push(',');
|
|
}
|
|
first = false;
|
|
out.push_str(&format!("{}:{}", json_escape(k), json_escape(v)));
|
|
}
|
|
out.push('}');
|
|
out
|
|
}
|
|
|
|
/// Serialize `s` as a complete JSON string literal (quotes included).
|
|
/// Delegates to the canonical op-util escaper.
|
|
pub(super) fn json_escape(s: &str) -> String {
|
|
op_util::json_escape::escape_json_quoted(s)
|
|
}
|