import { defineEventHandler, readBody, setResponseHeaders } from 'h3'; import { resolveClaudeCli } from '../../utils/resolve-claude-cli'; import { buildClaudeAgentEnv, buildSpawnClaudeCodeProcess, getClaudeAgentDebugFilePath, } from '../../utils/resolve-claude-agent-env'; import { writeFile, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { runCodexExec } from '../../utils/codex-client'; interface ValidateBody { system: string; message: string; imageBase64: string; model?: string; provider?: 'anthropic' | 'openai' | 'opencode' | 'gemini'; } /** * Vision-based validation endpoint. * Accepts a base64 PNG screenshot and a text prompt, sends multimodal * content blocks for analysis via Agent SDK. * * Saves screenshot to temp file, asks Claude Code to read it via its * built-in Read tool. */ export default defineEventHandler(async (event) => { const body = await readBody(event); if (!body?.system || !body?.message || !body?.imageBase64) { setResponseHeaders(event, { 'Content-Type': 'application/json' }); return { error: 'Missing required fields: system, message, imageBase64' }; } if (!body.model?.trim()) { setResponseHeaders(event, { 'Content-Type': 'application/json' }); return { error: 'Missing model. Model fallback is disabled.' }; } try { if (body.provider === 'anthropic') { return await validateViaAgentSDK(body, body.model); } if (body.provider === 'openai') { return await validateViaCodex(body, body.model); } if (body.provider === 'opencode') { return await validateViaOpenCode(body, body.model); } if (body.provider === 'gemini') { return await validateViaGemini(body, body.model); } // Builtin providers (MiniMax / DeepSeek / Bailian / Ark / etc) are // configured client-side and don't have vision support wired through // here yet. Most of the configured models in this category are text- // only anyway (MiniMax-M2.7, qwen3-coder-plus, ark-code-latest, // deepseek-v4-pro), so a vision call would fail at the upstream API // even if we did proxy it. Surface a clear `skipped` outcome so the // post-generation loop logs "[error] Analysis skipped (provider has // no vision support)" instead of the generic "unsupported provider" // string — that one looks like a config bug to the user. if (body.provider === 'builtin') { return { skipped: true, error: 'Vision validation is not supported for builtin providers (text-only models). ' + 'Connect Claude Code, Codex, OpenCode, or Gemini CLI for the vision self-check loop.', }; } return { error: 'Missing or unsupported provider. Provider fallback is disabled.' }; } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error'; return { error: message }; } }); function toImageBase64(data: string): string { const dataUrlPrefix = 'data:image/png;base64,'; return data.startsWith(dataUrlPrefix) ? data.slice(dataUrlPrefix.length) : data; } async function withTempImageFile( imageBase64: string, run: (tempPath: string) => Promise, insideProject = false, ): Promise { let tempDir: string; if (insideProject) { // Save inside the project directory so Claude Code Agent SDK (plan mode) // can read the file — it restricts reads to the project directory. const { mkdirSync, chmodSync } = await import('node:fs'); const baseDir = join(process.cwd(), '.openpencil-tmp'); mkdirSync(baseDir, { recursive: true }); chmodSync(baseDir, 0o700); tempDir = await mkdtemp(join(baseDir, 'validate-')); } else { tempDir = await mkdtemp(join(tmpdir(), 'openpencil-validate-')); } const tempPath = join(tempDir, 'screenshot.png'); try { await writeFile(tempPath, Buffer.from(toImageBase64(imageBase64), 'base64')); return await run(tempPath); } finally { await rm(tempDir, { recursive: true, force: true }).catch(() => {}); } } /** * Agent SDK: save screenshot to a temp PNG file inside the project directory, * then ask Claude Code to read it (Claude Code's Read tool supports images * natively). Must use insideProject=true because plan mode restricts reads * to the project directory. */ async function validateViaAgentSDK( body: ValidateBody, requestedModel?: string, ): Promise<{ text: string; skipped?: boolean; error?: string }> { return await withTempImageFile( body.imageBase64, async (tempPath) => { const { query } = await import('@anthropic-ai/claude-agent-sdk'); const env = buildClaudeAgentEnv(); const debugFile = getClaudeAgentDebugFilePath(); const claudePath = resolveClaudeCli(); const model = requestedModel; const prompt = `IMPORTANT: First, use the Read tool to read the image file at "${tempPath}". This is a PNG screenshot of a UI design. After viewing the image, analyze it according to these instructions: ${body.system} ${body.message} CRITICAL: Your ENTIRE response must be a single JSON object. No markdown, no explanation, no tool calls after reading the image. Just the JSON.`; const q = query({ prompt, options: { ...(model ? { model } : {}), maxTurns: 3, tools: [], plugins: [], permissionMode: 'plan', persistSession: false, env, ...(debugFile ? { debugFile } : {}), ...(claudePath ? { pathToClaudeCodeExecutable: claudePath } : {}), ...(buildSpawnClaudeCodeProcess() ? { spawnClaudeCodeProcess: buildSpawnClaudeCodeProcess() } : {}), }, }); try { for await (const message of q) { if (message.type === 'result') { const isErrorResult = 'is_error' in message && Boolean((message as { is_error?: boolean }).is_error); if (message.subtype === 'success' && !isErrorResult) { return { text: message.result }; } const errors = 'errors' in message ? (message.errors as string[]) : []; const resultText = 'result' in message ? String(message.result ?? '') : ''; return { error: errors.join('; ') || resultText || `Query ended with: ${message.subtype}`, text: '', }; } } } finally { q.close(); } return { text: '', skipped: true }; }, true, ); } async function validateViaCodex( body: ValidateBody, model?: string, ): Promise<{ text: string; skipped?: boolean; error?: string }> { return await withTempImageFile(body.imageBase64, async (tempPath) => { const result = await runCodexExec( `${body.message}\n\nOutput ONLY the JSON object, no markdown fences, no explanation.`, { model, systemPrompt: body.system, imageFiles: [tempPath], }, ); if (result.error) { return { text: '', error: result.error }; } return { text: result.text ?? '' }; }); } function parseOpenCodeModel(model?: string): { providerID: string; modelID: string } | undefined { if (!model || !model.includes('/')) return undefined; const idx = model.indexOf('/'); return { providerID: model.slice(0, idx), modelID: model.slice(idx + 1) }; } async function validateViaOpenCode( body: ValidateBody, model?: string, ): Promise<{ text: string; skipped?: boolean; error?: string }> { let ocServer: { close(): void } | undefined; try { const { getOpencodeClient } = await import('../../utils/opencode-client'); const oc = await getOpencodeClient(); const ocClient: any = oc.client; ocServer = oc.server; const { data: session, error: sessionError } = await ocClient.session.create({ title: 'OpenPencil Validate', }); if (sessionError || !session) { return { text: '', error: 'Failed to create OpenCode session' }; } await ocClient.session.prompt({ sessionID: session.id, noReply: true, parts: [{ type: 'text', text: body.system }], }); const parsed = parseOpenCodeModel(model); if (!parsed) { return { text: '', error: 'Invalid OpenCode model format. Expected "provider/model".' }; } const base64 = toImageBase64(body.imageBase64); const promptPayload = { sessionID: session.id, model: parsed, parts: [ { type: 'image', url: `data:image/png;base64,${base64}` }, { type: 'text', text: `${body.message}\n\nOutput ONLY the JSON object, no markdown fences, no explanation.`, }, ], }; const { data: result, error: promptError } = await ocClient.session.prompt(promptPayload); if (promptError) { return { text: '', error: 'OpenCode validation failed' }; } const texts: string[] = []; if (result?.parts) { for (const part of result.parts) { if (part.type === 'text' && part.text) { texts.push(part.text); } } } return { text: texts.join('') }; } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error'; return { text: '', error: message }; } finally { const { releaseOpencodeServer } = await import('../../utils/opencode-client'); releaseOpencodeServer(ocServer); } } /** Validate via Gemini CLI — saves screenshot to temp file, asks CLI to read it */ async function validateViaGemini( body: ValidateBody, model?: string, ): Promise<{ text: string; skipped?: boolean; error?: string }> { return await withTempImageFile(body.imageBase64, async (tempPath) => { const { runGeminiExec } = await import('../../utils/gemini-client'); const prompt = `Read the image file at "${tempPath}". This is a PNG screenshot of a UI design.\n\n${body.message}\n\nOutput ONLY the JSON object, no markdown fences, no explanation.`; const result = await runGeminiExec(prompt, { model, systemPrompt: body.system, }); if (result.error) { return { text: '', error: result.error }; } return { text: result.text ?? '' }; }); }