Merge pull request #507 from open-pencil/vision-workflow

feat(ai): add isolated visual inspection

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Added
 
 - Add local crash recovery for unsaved and pathless documents, including MCP-created documents. (#487)
+- Add isolated visual inspection that sends bounded selection renders to the configured Vision model and returns text findings without retaining image data in Design chat history. (#232, #471)
 - Allow supported AI model profiles to set a provider-specific reasoning effort. (#454)
 - Show unavailable or substituted document fonts with affected-layer selection and retry actions, and expose font fidelity through the Figma API and MCP tooling. (#503)
 

packages/core/src/tools/vector/export.ts

@@ -70,11 +70,19 @@ export const exportImage = defineTool({
     },
     scale: {
       type: 'number',
-      description: 'Export scale multiplier (default: 1)',
+      description: 'Export scale multiplier before the maximum-edge limit is applied (default: 1)',
       default: 1,
       min: 0.1,
       max: 4
     },
+    maxEdge: {
+      type: 'number',
+      description:
+        'Maximum output width or height in pixels. Preserves aspect ratio and never upscales. Defaults to 1280 for bounded model input.',
+      default: 1280,
+      min: 64,
+      max: 4096
+    },
     path: {
       type: 'string',
       description:
@@ -88,8 +96,33 @@ export const exportImage = defineTool({
     const ids =
       args.ids && args.ids.length > 0 ? args.ids : figma.currentPage.children.map((node) => node.id)
     const format = (args.format ?? 'PNG').toUpperCase() as RasterExportFormat
+    const requestedScale = args.scale ?? 1
+    const maxEdge = args.maxEdge ?? 1280
+    const nodes = ids.map((id) => figma.getNodeById(id)).filter((node) => node !== null)
+    if (nodes.length === 0) return { error: 'No visible nodes to export' }
+    const bounds = nodes.reduce(
+      (result, node) => {
+        const box = node.absoluteBoundingBox
+        const minX = Math.min(result.minX, box.x)
+        const minY = Math.min(result.minY, box.y)
+        const maxX = Math.max(result.maxX, box.x + box.width)
+        const maxY = Math.max(result.maxY, box.y + box.height)
+        return { minX, minY, maxX, maxY }
+      },
+      {
+        minX: Number.POSITIVE_INFINITY,
+        minY: Number.POSITIVE_INFINITY,
+        maxX: Number.NEGATIVE_INFINITY,
+        maxY: Number.NEGATIVE_INFINITY
+      }
+    )
+    const width = bounds.maxX - bounds.minX
+    const height = bounds.maxY - bounds.minY
+    const longestEdge = Math.max(width, height)
+    const boundedScale = longestEdge > 0 ? Math.min(requestedScale, maxEdge / longestEdge) : 0
+    if (boundedScale <= 0) return { error: 'No visible nodes to export' }
     const data = await figma.exportImage(ids, {
-      scale: args.scale ?? 1,
+      scale: boundedScale,
       format
     })
     if (!data || data.length === 0) return { error: 'No visible nodes to export' }
@@ -98,7 +131,10 @@ export const exportImage = defineTool({
     return {
       mimeType: mimeMap[format],
       base64,
-      byteLength: data.length
+      byteLength: data.length,
+      width: Math.ceil(width * boundedScale),
+      height: Math.ceil(height * boundedScale),
+      scale: boundedScale
     }
   }
 })

src/app/ai/chat/reasoning.ts

@@ -0,0 +1,18 @@
+import type { AIProviderID } from '@open-pencil/core/constants'
+
+type JSONValue = null | boolean | number | string | JSONValue[] | { [key: string]: JSONValue }
+export type AIProviderOptions = Record<string, { [key: string]: JSONValue }>
+
+export function buildReasoningProviderOptions(
+  providerID: AIProviderID,
+  reasoningEffort: string
+): AIProviderOptions | undefined {
+  if (!reasoningEffort) return undefined
+  if (providerID === 'openrouter') {
+    return { openrouter: { reasoning: { effort: reasoningEffort } } }
+  }
+  if (providerID === 'openai' || providerID === 'openai-compatible') {
+    return { openai: { reasoningEffort } }
+  }
+  return undefined
+}

src/app/ai/chat/system-prompt.md

@@ -211,7 +211,7 @@ Common warnings:
 
 ⚠ **Don't call `viewport_zoom_to_fit` or `describe` with the same arguments as a previous call in the same conversation.** Check your last calls before repeating.
 
-🚫 **Never use `export_image`** — slow and wastes tokens. Use `describe` instead.
+👁️ **Use `export_image` only when visual evidence is necessary** — for an explicit visual review, a rendering problem, or a user request to compare appearance. Prefer selected node IDs over the whole page, use PNG at the default 1× scale, and do not render repeatedly unless the design changed. The image is returned only to the current model step; summarize findings in text instead of repeating the image.
 
 ## Step budget
 

src/app/ai/chat/transports.ts

@@ -13,6 +13,7 @@ import {
   type AIChatFailure
 } from '@/app/ai/chat/failure'
 import { resolveLanguageModelID } from '@/app/ai/chat/model'
+import { buildReasoningProviderOptions, type AIProviderOptions } from '@/app/ai/chat/reasoning'
 import SYSTEM_PROMPT from '@/app/ai/chat/system-prompt.md?raw'
 import { createAIModelRuntime } from '@/app/ai/models'
 import { MAX_AGENT_STEPS, createAITools, recordStepUsage, resetRunSteps } from '@/app/ai/tools'
@@ -49,23 +50,6 @@ function supportsAnthropicCaching(providerID: AIProviderID, modelID: string): bo
   )
 }
 
-type JSONValue = null | boolean | number | string | JSONValue[] | { [key: string]: JSONValue }
-type AIProviderOptions = Record<string, { [key: string]: JSONValue }>
-
-export function buildReasoningProviderOptions(
-  providerID: AIProviderID,
-  reasoningEffort: string
-): AIProviderOptions | undefined {
-  if (!reasoningEffort) return undefined
-  if (providerID === 'openrouter') {
-    return { openrouter: { reasoning: { effort: reasoningEffort } } }
-  }
-  if (providerID === 'openai' || providerID === 'openai-compatible') {
-    return { openai: { reasoningEffort } }
-  }
-  return undefined
-}
-
 function mergeProviderOptions(
   cacheOptions: typeof ANTHROPIC_CACHE_CONTROL | undefined,
   reasoningOptions: AIProviderOptions | undefined

src/app/ai/tools/index.ts

@@ -3,7 +3,7 @@ import { tool } from 'ai'
 import * as v from 'valibot'
 
 import { computeAllLayouts } from '@open-pencil/core/layout'
-import { CORE_TOOLS, toolsToAI } from '@open-pencil/core/tools'
+import { CORE_TOOLS, EXTENDED_TOOLS, toolsToAI } from '@open-pencil/core/tools'
 import type { StepBudget, ToolLogEntry } from '@open-pencil/core/tools'
 import type { SceneNode } from '@open-pencil/scene-graph'
 
@@ -12,8 +12,16 @@ import { getActiveEditorStore } from '@/app/editor/active-store'
 import type { EditorStore } from '@/app/editor/active-store'
 import { ensureGraphFonts } from '@/app/editor/fonts'
 
+import { createVisualInspectionTool } from './vision'
+
 export const MAX_AGENT_STEPS = 50
 
+const VISUAL_INSPECTION_TOOL_NAMES = new Set(['export_image'])
+const AI_CHAT_TOOLS = [
+  ...CORE_TOOLS,
+  ...EXTENDED_TOOLS.filter((definition) => VISUAL_INSPECTION_TOOL_NAMES.has(definition.name))
+]
+
 export interface StepUsage {
   inputTokens: number
   outputTokens: number
@@ -86,50 +94,53 @@ export function createAITools(store: EditorStore) {
   let beforeSnapshot: Map<string, SceneNode> | null = null
   const runState = getRunState(store)
 
-  return toolsToAI(
-    CORE_TOOLS,
-    {
-      getFigma: () => makeFigmaFromStore(store),
-      onBeforeExecute: (def) => {
-        if (def.mutates) {
-          beforeSnapshot = store.snapshotPage()
-        }
-      },
-      onAfterExecute: async (def) => {
-        if (def.mutates) {
-          const pageId = store.state.currentPageId
-          const pageNode = store.graph.getNode(pageId)
-          if (pageNode) await ensureGraphFonts(store.graph, pageNode.childIds, store.renderer)
-          computeAllLayouts(store.graph, pageId)
-          store.requestRender()
-          if (beforeSnapshot) {
-            const before = beforeSnapshot
-            const after = store.snapshotPage()
-            store.pushUndoEntry({
-              label: `AI: ${def.name}`,
-              forward: () => store.restorePageFromSnapshot(after),
-              inverse: () => store.restorePageFromSnapshot(before)
-            })
-            beforeSnapshot = null
+  return {
+    ...toolsToAI(
+      AI_CHAT_TOOLS,
+      {
+        getFigma: () => makeFigmaFromStore(store),
+        onBeforeExecute: (def) => {
+          if (def.mutates) {
+            beforeSnapshot = store.snapshotPage()
           }
-        }
+        },
+        onAfterExecute: async (def) => {
+          if (def.mutates) {
+            const pageId = store.state.currentPageId
+            const pageNode = store.graph.getNode(pageId)
+            if (pageNode) await ensureGraphFonts(store.graph, pageNode.childIds, store.renderer)
+            computeAllLayouts(store.graph, pageId)
+            store.requestRender()
+            if (beforeSnapshot) {
+              const before = beforeSnapshot
+              const after = store.snapshotPage()
+              store.pushUndoEntry({
+                label: `AI: ${def.name}`,
+                forward: () => store.restorePageFromSnapshot(after),
+                inverse: () => store.restorePageFromSnapshot(before)
+              })
+              beforeSnapshot = null
+            }
+          }
+        },
+        onFlashNodes: (nodeIds) => {
+          store.renderer?.aiClearActive()
+          if (nodeIds.length > 0) {
+            store.aiFlashDone(nodeIds)
+          }
+        },
+        onToolLog: (entry) => {
+          runState.toolLog.push(entry)
+        },
+        getStepBudget: (): StepBudget => ({
+          current: runState.currentSteps,
+          max: MAX_AGENT_STEPS
+        })
       },
-      onFlashNodes: (nodeIds) => {
-        store.renderer?.aiClearActive()
-        if (nodeIds.length > 0) {
-          store.aiFlashDone(nodeIds)
-        }
-      },
-      onToolLog: (entry) => {
-        runState.toolLog.push(entry)
-      },
-      getStepBudget: (): StepBudget => ({
-        current: runState.currentSteps,
-        max: MAX_AGENT_STEPS
-      })
-    },
-    { v, valibotSchema, tool }
-  )
+      { v, valibotSchema, tool }
+    ),
+    inspect_visual: createVisualInspectionTool(store)
+  }
 }
 
 export type AITools = ReturnType<typeof createAITools>

src/app/ai/tools/vision.ts

@@ -0,0 +1,117 @@
+import { valibotSchema } from '@ai-sdk/valibot'
+import { generateText, tool } from 'ai'
+import * as v from 'valibot'
+
+import { computeContentBounds } from '@open-pencil/core/io'
+
+import { buildReasoningProviderOptions } from '@/app/ai/chat/reasoning'
+import { createAIModelRuntime } from '@/app/ai/models'
+import type { EditorStore } from '@/app/editor/active-store'
+
+const DEFAULT_VISION_MAX_EDGE = 1280
+const MAX_VISION_MAX_EDGE = 4096
+const MAX_VISION_OUTPUT_TOKENS = 1200
+
+export type VisualInspectionRequest = {
+  ids?: string[]
+  question?: string
+  maxEdge?: number
+}
+
+export type VisualInspectionResult = {
+  analysis: string
+  inspectedNodeIds: string[]
+  image: { width: number; height: number }
+}
+
+export type VisualInspectionDependencies = {
+  createRuntime: typeof createAIModelRuntime
+  inspect: typeof generateText
+}
+
+export function boundedImageScale(
+  width: number,
+  height: number,
+  maxEdge = DEFAULT_VISION_MAX_EDGE
+): number {
+  const longestEdge = Math.max(width, height)
+  if (longestEdge <= 0) return 0
+  return Math.min(1, maxEdge / longestEdge)
+}
+
+export async function inspectRenderedDesign(
+  store: EditorStore,
+  request: VisualInspectionRequest,
+  dependencies: VisualInspectionDependencies = {
+    createRuntime: createAIModelRuntime,
+    inspect: generateText
+  }
+): Promise<VisualInspectionResult | { error: string }> {
+  const runtime = await dependencies.createRuntime('vision')
+  if (runtime?.kind !== 'direct') {
+    return { error: 'Configure a vision-capable model in Settings to inspect rendered designs.' }
+  }
+
+  const pageId = store.state.currentPageId
+  let nodeIds = request.ids ?? []
+  if (nodeIds.length === 0) nodeIds = [...store.state.selectedIds]
+  if (nodeIds.length === 0) {
+    nodeIds = store.graph.getChildren(pageId).map((node) => node.id)
+  }
+  const bounds = computeContentBounds(store.graph, nodeIds)
+  if (!bounds) return { error: 'No visible design content to inspect.' }
+  const width = bounds.maxX - bounds.minX
+  const height = bounds.maxY - bounds.minY
+  const scale = boundedImageScale(width, height, request.maxEdge)
+  if (scale <= 0) return { error: 'No visible design content to inspect.' }
+
+  const image = await store.renderExportImage(nodeIds, scale, 'PNG', pageId)
+  if (!image) return { error: 'Could not render the design for visual inspection.' }
+
+  const result = await dependencies.inspect({
+    model: runtime.model,
+    maxOutputTokens: Math.min(runtime.role.profile.maxOutputTokens, MAX_VISION_OUTPUT_TOKENS),
+    providerOptions: buildReasoningProviderOptions(
+      runtime.role.connection.providerID,
+      runtime.role.profile.reasoningEffort ?? ''
+    ),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text:
+              request.question?.trim() ||
+              'Review this rendered design. Concisely identify visual hierarchy, alignment, spacing, clipping, contrast, and rendering problems. Return actionable findings only.'
+          },
+          { type: 'file', mediaType: 'image/png', data: image }
+        ]
+      }
+    ]
+  })
+
+  return {
+    analysis: result.text,
+    inspectedNodeIds: nodeIds,
+    image: {
+      width: Math.ceil(width * scale),
+      height: Math.ceil(height * scale)
+    }
+  }
+}
+
+export function createVisualInspectionTool(store: EditorStore) {
+  return tool({
+    description:
+      'Render the current selection or specified nodes and ask the isolated Vision model to inspect their visual appearance. Returns text findings only; the image is not added to the Design chat history. Use only for explicit visual review or rendering diagnosis.',
+    inputSchema: valibotSchema(
+      v.object({
+        ids: v.optional(v.array(v.string())),
+        question: v.optional(v.string()),
+        maxEdge: v.optional(v.pipe(v.number(), v.minValue(64), v.maxValue(MAX_VISION_MAX_EDGE)))
+      })
+    ),
+    execute: (request) => inspectRenderedDesign(store, request)
+  })
+}

tests/engine/app/ai/model-roles.test.ts

@@ -1,6 +1,6 @@
 import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
 
-import { buildReasoningProviderOptions } from '@/app/ai/chat/transports'
+import { buildReasoningProviderOptions } from '@/app/ai/chat/reasoning'
 import {
   aiModelSettings,
   createAIModelRuntime,

tests/engine/app/ai/vision.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, test } from 'bun:test'
+
+import type { LanguageModel } from 'ai'
+
+import { SceneGraph } from '@open-pencil/scene-graph'
+
+import {
+  boundedImageScale,
+  inspectRenderedDesign,
+  type VisualInspectionDependencies
+} from '@/app/ai/tools/vision'
+import type { EditorStore } from '@/app/editor/session/create'
+
+describe('isolated visual inspection', () => {
+  test('bounds renders and forwards only textual findings to the caller', async () => {
+    const graph = new SceneGraph()
+    const page = graph.getPages()[0]
+    const frame = graph.createNode('FRAME', page.id, { width: 2560, height: 1600 })
+    const rendered: Array<{ ids: string[]; scale: number }> = []
+    const inspections: unknown[] = []
+    const store = {
+      graph,
+      state: { currentPageId: page.id, selectedIds: new Set([frame.id]) },
+      renderExportImage: async (ids: string[], scale: number) => {
+        rendered.push({ ids, scale })
+        return new Uint8Array([1, 2, 3])
+      }
+    } as EditorStore
+    const dependencies: VisualInspectionDependencies = {
+      createRuntime: async () =>
+        ({
+          kind: 'direct',
+          model: {} as LanguageModel,
+          role: {
+            requestedRole: 'vision',
+            profile: { maxOutputTokens: 8000, reasoningEffort: 'low' },
+            connection: { providerID: 'openrouter' }
+          }
+        }) as never,
+      inspect: async (options) => {
+        inspections.push(options)
+        return { text: 'Align the button with the form edge.' } as never
+      }
+    }
+
+    const result = await inspectRenderedDesign(store, {}, dependencies)
+
+    expect(rendered).toEqual([{ ids: [frame.id], scale: 0.5 }])
+    expect(result).toEqual({
+      analysis: 'Align the button with the form edge.',
+      inspectedNodeIds: [frame.id],
+      image: { width: 1280, height: 800 }
+    })
+    const request = inspections[0] as {
+      maxOutputTokens: number
+      providerOptions?: unknown
+      messages: Array<{ content: Array<{ type: string; data?: Uint8Array }> }>
+    }
+    expect(request.maxOutputTokens).toBe(1200)
+    expect(request.providerOptions).toEqual({ openrouter: { reasoning: { effort: 'low' } } })
+    expect(request.messages[0]?.content[1]?.type).toBe('file')
+    expect(request.messages[0]?.content[1]?.data).toEqual(new Uint8Array([1, 2, 3]))
+    expect(result).not.toHaveProperty('base64')
+  })
+
+  test('never upscales bounded images', () => {
+    expect(boundedImageScale(640, 400)).toBe(1)
+    expect(boundedImageScale(2560, 1600)).toBe(0.5)
+  })
+})

tests/engine/tools/export-image.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test } from 'bun:test'
+
+import { getTool, setupToolTest } from '#tests/helpers/tools'
+
+type ExportImageResult = {
+  width?: unknown
+  height?: unknown
+  scale?: unknown
+}
+
+function exportImageResult(value: unknown): ExportImageResult {
+  return typeof value === 'object' && value !== null ? value : {}
+}
+
+describe('export_image tool', () => {
+  test('bounds model-facing image output by its longest edge without upscaling', async () => {
+    const { figma } = setupToolTest()
+    const frame = figma.createFrame()
+    frame.resize(2560, 1600)
+    const calls: Array<{ scale?: number; format?: string }> = []
+    figma.exportImage = async (_ids, options) => {
+      calls.push(options)
+      return new Uint8Array([1, 2, 3])
+    }
+
+    const result = exportImageResult(
+      await getTool('export_image').execute(figma, {
+        ids: [frame.id],
+        format: 'PNG',
+        scale: 2,
+        maxEdge: 1280
+      })
+    )
+
+    expect(calls).toEqual([{ scale: 0.5, format: 'PNG' }])
+    expect(result.width).toBe(1280)
+    expect(result.height).toBe(800)
+    expect(result.scale).toBe(0.5)
+  })
+
+  test('keeps the requested scale when the image already fits', async () => {
+    const { figma } = setupToolTest()
+    const frame = figma.createFrame()
+    frame.resize(640, 400)
+    const calls: Array<{ scale?: number; format?: string }> = []
+    figma.exportImage = async (_ids, options) => {
+      calls.push(options)
+      return new Uint8Array([1])
+    }
+
+    await getTool('export_image').execute(figma, {
+      ids: [frame.id],
+      scale: 1,
+      maxEdge: 1280
+    })
+
+    expect(calls[0]?.scale).toBe(1)
+  })
+})