From 8294e32f33982c08e3cd6841b2d7662cdf49454b Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Thu, 13 Aug 2026 20:55:40 +0300 Subject: [PATCH] feat(ai): add isolated visual inspection - Route bounded selection renders through the optional Vision model - Return compact text findings without retaining image data in Design chat history - Bound MCP and AI image exports by their longest pixel edge --- CHANGELOG.md | 1 + packages/core/src/tools/vector/export.ts | 42 ++++++++- src/app/ai/chat/system-prompt.md | 2 +- src/app/ai/tools/index.ts | 97 +++++++++++--------- src/app/ai/tools/vision.ts | 112 +++++++++++++++++++++++ tests/engine/app/ai/vision.test.ts | 68 ++++++++++++++ tests/engine/tools/export-image.test.ts | 59 ++++++++++++ 7 files changed, 334 insertions(+), 47 deletions(-) create mode 100644 src/app/ai/tools/vision.ts create mode 100644 tests/engine/app/ai/vision.test.ts create mode 100644 tests/engine/tools/export-image.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 09f6ba0cb..1049eac36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Add local crash recovery for unsaved and pathless documents, including MCP-created documents. (#487) +- Add isolated visual inspection that sends bounded selection renders to the configured Vision model and returns text findings without retaining image data in Design chat history. (#232, #471) - Allow supported AI model profiles to set a provider-specific reasoning effort. (#454) - Show unavailable or substituted document fonts with affected-layer selection and retry actions, and expose font fidelity through the Figma API and MCP tooling. (#503) diff --git a/packages/core/src/tools/vector/export.ts b/packages/core/src/tools/vector/export.ts index faeea55b5..cb92a5195 100644 --- a/packages/core/src/tools/vector/export.ts +++ b/packages/core/src/tools/vector/export.ts @@ -70,11 +70,19 @@ export const exportImage = defineTool({ }, scale: { type: 'number', - description: 'Export scale multiplier (default: 1)', + description: 'Export scale multiplier before the maximum-edge limit is applied (default: 1)', default: 1, min: 0.1, max: 4 }, + maxEdge: { + type: 'number', + description: + 'Maximum output width or height in pixels. Preserves aspect ratio and never upscales. Defaults to 1280 for bounded model input.', + default: 1280, + min: 64, + max: 4096 + }, path: { type: 'string', description: @@ -88,8 +96,33 @@ export const exportImage = defineTool({ const ids = args.ids && args.ids.length > 0 ? args.ids : figma.currentPage.children.map((node) => node.id) const format = (args.format ?? 'PNG').toUpperCase() as RasterExportFormat + const requestedScale = args.scale ?? 1 + const maxEdge = args.maxEdge ?? 1280 + const nodes = ids.map((id) => figma.getNodeById(id)).filter((node) => node !== null) + if (nodes.length === 0) return { error: 'No visible nodes to export' } + const bounds = nodes.reduce( + (result, node) => { + const box = node.absoluteBoundingBox + const minX = Math.min(result.minX, box.x) + const minY = Math.min(result.minY, box.y) + const maxX = Math.max(result.maxX, box.x + box.width) + const maxY = Math.max(result.maxY, box.y + box.height) + return { minX, minY, maxX, maxY } + }, + { + minX: Number.POSITIVE_INFINITY, + minY: Number.POSITIVE_INFINITY, + maxX: Number.NEGATIVE_INFINITY, + maxY: Number.NEGATIVE_INFINITY + } + ) + const width = bounds.maxX - bounds.minX + const height = bounds.maxY - bounds.minY + const longestEdge = Math.max(width, height) + const boundedScale = longestEdge > 0 ? Math.min(requestedScale, maxEdge / longestEdge) : 0 + if (boundedScale <= 0) return { error: 'No visible nodes to export' } const data = await figma.exportImage(ids, { - scale: args.scale ?? 1, + scale: boundedScale, format }) if (!data || data.length === 0) return { error: 'No visible nodes to export' } @@ -98,7 +131,10 @@ export const exportImage = defineTool({ return { mimeType: mimeMap[format], base64, - byteLength: data.length + byteLength: data.length, + width: Math.ceil(width * boundedScale), + height: Math.ceil(height * boundedScale), + scale: boundedScale } } }) diff --git a/src/app/ai/chat/system-prompt.md b/src/app/ai/chat/system-prompt.md index 5870ef673..c897044fd 100644 --- a/src/app/ai/chat/system-prompt.md +++ b/src/app/ai/chat/system-prompt.md @@ -211,7 +211,7 @@ Common warnings: ⚠ **Don't call `viewport_zoom_to_fit` or `describe` with the same arguments as a previous call in the same conversation.** Check your last calls before repeating. -🚫 **Never use `export_image`** — slow and wastes tokens. Use `describe` instead. +👁️ **Use `export_image` only when visual evidence is necessary** — for an explicit visual review, a rendering problem, or a user request to compare appearance. Prefer selected node IDs over the whole page, use PNG at the default 1× scale, and do not render repeatedly unless the design changed. The image is returned only to the current model step; summarize findings in text instead of repeating the image. ## Step budget diff --git a/src/app/ai/tools/index.ts b/src/app/ai/tools/index.ts index 6cbdfd1f4..47d56dd5d 100644 --- a/src/app/ai/tools/index.ts +++ b/src/app/ai/tools/index.ts @@ -3,7 +3,7 @@ import { tool } from 'ai' import * as v from 'valibot' import { computeAllLayouts } from '@open-pencil/core/layout' -import { CORE_TOOLS, toolsToAI } from '@open-pencil/core/tools' +import { CORE_TOOLS, EXTENDED_TOOLS, toolsToAI } from '@open-pencil/core/tools' import type { StepBudget, ToolLogEntry } from '@open-pencil/core/tools' import type { SceneNode } from '@open-pencil/scene-graph' @@ -12,8 +12,16 @@ import { getActiveEditorStore } from '@/app/editor/active-store' import type { EditorStore } from '@/app/editor/active-store' import { ensureGraphFonts } from '@/app/editor/fonts' +import { createVisualInspectionTool } from './vision' + export const MAX_AGENT_STEPS = 50 +const VISUAL_INSPECTION_TOOL_NAMES = new Set(['export_image']) +const AI_CHAT_TOOLS = [ + ...CORE_TOOLS, + ...EXTENDED_TOOLS.filter((definition) => VISUAL_INSPECTION_TOOL_NAMES.has(definition.name)) +] + export interface StepUsage { inputTokens: number outputTokens: number @@ -86,50 +94,53 @@ export function createAITools(store: EditorStore) { let beforeSnapshot: Map | null = null const runState = getRunState(store) - return toolsToAI( - CORE_TOOLS, - { - getFigma: () => makeFigmaFromStore(store), - onBeforeExecute: (def) => { - if (def.mutates) { - beforeSnapshot = store.snapshotPage() - } - }, - onAfterExecute: async (def) => { - if (def.mutates) { - const pageId = store.state.currentPageId - const pageNode = store.graph.getNode(pageId) - if (pageNode) await ensureGraphFonts(store.graph, pageNode.childIds, store.renderer) - computeAllLayouts(store.graph, pageId) - store.requestRender() - if (beforeSnapshot) { - const before = beforeSnapshot - const after = store.snapshotPage() - store.pushUndoEntry({ - label: `AI: ${def.name}`, - forward: () => store.restorePageFromSnapshot(after), - inverse: () => store.restorePageFromSnapshot(before) - }) - beforeSnapshot = null + return { + ...toolsToAI( + AI_CHAT_TOOLS, + { + getFigma: () => makeFigmaFromStore(store), + onBeforeExecute: (def) => { + if (def.mutates) { + beforeSnapshot = store.snapshotPage() } - } + }, + onAfterExecute: async (def) => { + if (def.mutates) { + const pageId = store.state.currentPageId + const pageNode = store.graph.getNode(pageId) + if (pageNode) await ensureGraphFonts(store.graph, pageNode.childIds, store.renderer) + computeAllLayouts(store.graph, pageId) + store.requestRender() + if (beforeSnapshot) { + const before = beforeSnapshot + const after = store.snapshotPage() + store.pushUndoEntry({ + label: `AI: ${def.name}`, + forward: () => store.restorePageFromSnapshot(after), + inverse: () => store.restorePageFromSnapshot(before) + }) + beforeSnapshot = null + } + } + }, + onFlashNodes: (nodeIds) => { + store.renderer?.aiClearActive() + if (nodeIds.length > 0) { + store.aiFlashDone(nodeIds) + } + }, + onToolLog: (entry) => { + runState.toolLog.push(entry) + }, + getStepBudget: (): StepBudget => ({ + current: runState.currentSteps, + max: MAX_AGENT_STEPS + }) }, - onFlashNodes: (nodeIds) => { - store.renderer?.aiClearActive() - if (nodeIds.length > 0) { - store.aiFlashDone(nodeIds) - } - }, - onToolLog: (entry) => { - runState.toolLog.push(entry) - }, - getStepBudget: (): StepBudget => ({ - current: runState.currentSteps, - max: MAX_AGENT_STEPS - }) - }, - { v, valibotSchema, tool } - ) + { v, valibotSchema, tool } + ), + inspect_visual: createVisualInspectionTool(store) + } } export type AITools = ReturnType diff --git a/src/app/ai/tools/vision.ts b/src/app/ai/tools/vision.ts new file mode 100644 index 000000000..3027ae2a1 --- /dev/null +++ b/src/app/ai/tools/vision.ts @@ -0,0 +1,112 @@ +import { valibotSchema } from '@ai-sdk/valibot' +import { generateText, tool } from 'ai' +import * as v from 'valibot' + +import { computeContentBounds } from '@open-pencil/core/io' + +import { createAIModelRuntime } from '@/app/ai/models' +import type { EditorStore } from '@/app/editor/active-store' + +const DEFAULT_VISION_MAX_EDGE = 1280 +const MAX_VISION_MAX_EDGE = 4096 +const MAX_VISION_OUTPUT_TOKENS = 1200 + +export type VisualInspectionRequest = { + ids?: string[] + question?: string + maxEdge?: number +} + +export type VisualInspectionResult = { + analysis: string + inspectedNodeIds: string[] + image: { width: number; height: number } +} + +export type VisualInspectionDependencies = { + createRuntime: typeof createAIModelRuntime + inspect: typeof generateText +} + +export function boundedImageScale( + width: number, + height: number, + maxEdge = DEFAULT_VISION_MAX_EDGE +): number { + const longestEdge = Math.max(width, height) + if (longestEdge <= 0) return 0 + return Math.min(1, maxEdge / longestEdge) +} + +export async function inspectRenderedDesign( + store: EditorStore, + request: VisualInspectionRequest, + dependencies: VisualInspectionDependencies = { + createRuntime: createAIModelRuntime, + inspect: generateText + } +): Promise { + const runtime = await dependencies.createRuntime('vision') + if (runtime?.kind !== 'direct') { + return { error: 'Configure a vision-capable model in Settings to inspect rendered designs.' } + } + + const pageId = store.state.currentPageId + let nodeIds = request.ids ?? [] + if (nodeIds.length === 0) nodeIds = [...store.state.selectedIds] + if (nodeIds.length === 0) { + nodeIds = store.graph.getChildren(pageId).map((node) => node.id) + } + const bounds = computeContentBounds(store.graph, nodeIds) + if (!bounds) return { error: 'No visible design content to inspect.' } + const width = bounds.maxX - bounds.minX + const height = bounds.maxY - bounds.minY + const scale = boundedImageScale(width, height, request.maxEdge) + if (scale <= 0) return { error: 'No visible design content to inspect.' } + + const image = await store.renderExportImage(nodeIds, scale, 'PNG', pageId) + if (!image) return { error: 'Could not render the design for visual inspection.' } + + const result = await dependencies.inspect({ + model: runtime.model, + maxOutputTokens: Math.min(runtime.role.profile.maxOutputTokens, MAX_VISION_OUTPUT_TOKENS), + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: + request.question?.trim() || + 'Review this rendered design. Concisely identify visual hierarchy, alignment, spacing, clipping, contrast, and rendering problems. Return actionable findings only.' + }, + { type: 'file', mediaType: 'image/png', data: image } + ] + } + ] + }) + + return { + analysis: result.text, + inspectedNodeIds: nodeIds, + image: { + width: Math.ceil(width * scale), + height: Math.ceil(height * scale) + } + } +} + +export function createVisualInspectionTool(store: EditorStore) { + return tool({ + description: + 'Render the current selection or specified nodes and ask the isolated Vision model to inspect their visual appearance. Returns text findings only; the image is not added to the Design chat history. Use only for explicit visual review or rendering diagnosis.', + inputSchema: valibotSchema( + v.object({ + ids: v.optional(v.array(v.string())), + question: v.optional(v.string()), + maxEdge: v.optional(v.pipe(v.number(), v.minValue(64), v.maxValue(MAX_VISION_MAX_EDGE))) + }) + ), + execute: (request) => inspectRenderedDesign(store, request) + }) +} diff --git a/tests/engine/app/ai/vision.test.ts b/tests/engine/app/ai/vision.test.ts new file mode 100644 index 000000000..b4db6ff22 --- /dev/null +++ b/tests/engine/app/ai/vision.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from 'bun:test' + +import type { LanguageModel } from 'ai' + +import { SceneGraph } from '@open-pencil/scene-graph' + +import { + boundedImageScale, + inspectRenderedDesign, + type VisualInspectionDependencies +} from '@/app/ai/tools/vision' +import type { EditorStore } from '@/app/editor/session/create' + +describe('isolated visual inspection', () => { + test('bounds renders and forwards only textual findings to the caller', async () => { + const graph = new SceneGraph() + const page = graph.getPages()[0] + const frame = graph.createNode('FRAME', page.id, { width: 2560, height: 1600 }) + const rendered: Array<{ ids: string[]; scale: number }> = [] + const inspections: unknown[] = [] + const store = { + graph, + state: { currentPageId: page.id, selectedIds: new Set([frame.id]) }, + renderExportImage: async (ids: string[], scale: number) => { + rendered.push({ ids, scale }) + return new Uint8Array([1, 2, 3]) + } + } as EditorStore + const dependencies: VisualInspectionDependencies = { + createRuntime: async () => + ({ + kind: 'direct', + model: {} as LanguageModel, + role: { + requestedRole: 'vision', + profile: { maxOutputTokens: 8000 }, + connection: {} + } + }) as never, + inspect: async (options) => { + inspections.push(options) + return { text: 'Align the button with the form edge.' } as never + } + } + + const result = await inspectRenderedDesign(store, {}, dependencies) + + expect(rendered).toEqual([{ ids: [frame.id], scale: 0.5 }]) + expect(result).toEqual({ + analysis: 'Align the button with the form edge.', + inspectedNodeIds: [frame.id], + image: { width: 1280, height: 800 } + }) + const request = inspections[0] as { + maxOutputTokens: number + messages: Array<{ content: Array<{ type: string; data?: Uint8Array }> }> + } + expect(request.maxOutputTokens).toBe(1200) + expect(request.messages[0]?.content[1]?.type).toBe('file') + expect(request.messages[0]?.content[1]?.data).toEqual(new Uint8Array([1, 2, 3])) + expect(result).not.toHaveProperty('base64') + }) + + test('never upscales bounded images', () => { + expect(boundedImageScale(640, 400)).toBe(1) + expect(boundedImageScale(2560, 1600)).toBe(0.5) + }) +}) diff --git a/tests/engine/tools/export-image.test.ts b/tests/engine/tools/export-image.test.ts new file mode 100644 index 000000000..d07be795e --- /dev/null +++ b/tests/engine/tools/export-image.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from 'bun:test' + +import { getTool, setupToolTest } from '#tests/helpers/tools' + +type ExportImageResult = { + width?: unknown + height?: unknown + scale?: unknown +} + +function exportImageResult(value: unknown): ExportImageResult { + return typeof value === 'object' && value !== null ? value : {} +} + +describe('export_image tool', () => { + test('bounds model-facing image output by its longest edge without upscaling', async () => { + const { figma } = setupToolTest() + const frame = figma.createFrame() + frame.resize(2560, 1600) + const calls: Array<{ scale?: number; format?: string }> = [] + figma.exportImage = async (_ids, options) => { + calls.push(options) + return new Uint8Array([1, 2, 3]) + } + + const result = exportImageResult( + await getTool('export_image').execute(figma, { + ids: [frame.id], + format: 'PNG', + scale: 2, + maxEdge: 1280 + }) + ) + + expect(calls).toEqual([{ scale: 0.5, format: 'PNG' }]) + expect(result.width).toBe(1280) + expect(result.height).toBe(800) + expect(result.scale).toBe(0.5) + }) + + test('keeps the requested scale when the image already fits', async () => { + const { figma } = setupToolTest() + const frame = figma.createFrame() + frame.resize(640, 400) + const calls: Array<{ scale?: number; format?: string }> = [] + figma.exportImage = async (_ids, options) => { + calls.push(options) + return new Uint8Array([1]) + } + + await getTool('export_image').execute(figma, { + ids: [frame.id], + scale: 1, + maxEdge: 1280 + }) + + expect(calls[0]?.scale).toBe(1) + }) +})