feat(ai): enhance design generation with variable support and context inclusion

- Update AI design generation functions to incorporate document variables and themes, allowing for dynamic references in generated designs.
- Modify the buildContextString function to include variable summaries for improved context during AI interactions.
- Enhance prompts to instruct the AI on using variable references instead of hardcoded values, ensuring consistency and flexibility in design outputs.
- Refactor related types and services to support the new variable and theme structures, improving overall design generation capabilities.
This commit is contained in:
Kayshen-X 2026-02-20 15:54:26 +08:00
parent 7bfcb94f9a
commit 24fb09f386
4 changed files with 84 additions and 6 deletions

View file

@ -74,7 +74,8 @@ function isDesignRequest(text: string): boolean {
function buildContextString(): string {
const selectedIds = useCanvasStore.getState().selection.selectedIds
const flatNodes = useDocumentStore.getState().getFlatNodes()
const { getFlatNodes, document: doc } = useDocumentStore.getState()
const flatNodes = getFlatNodes()
const parts: string[] = []
@ -96,6 +97,14 @@ function buildContextString(): string {
parts.push(`Selected: ${selectedSummary}`)
}
// Include variable summary so chat mode also knows about design tokens
if (doc.variables && Object.keys(doc.variables).length > 0) {
const varNames = Object.entries(doc.variables)
.map(([n, d]) => `$${n}(${d.type})`)
.join(', ')
parts.push(`Variables: ${varNames}`)
}
return parts.length > 0 ? `\n\n[Canvas context: ${parts.join('. ')}]` : ''
}
@ -164,14 +173,17 @@ function useChatHandlers() {
if (isDesign) {
if (isModification) {
// --- MODIFICATION MODE ---
const { getNodeById } = useDocumentStore.getState()
const { getNodeById, document: modDoc } = useDocumentStore.getState()
const selectedNodes = selectedIds.map(id => getNodeById(id)).filter(Boolean) as any[]
// We update the UI to show we are working
accumulated = '<step title="Checking guidelines">Analyzing modification request...</step>'
updateLastMessage(accumulated)
const { rawResponse, nodes } = await generateDesignModification(selectedNodes, messageText)
const { rawResponse, nodes } = await generateDesignModification(selectedNodes, messageText, {
variables: modDoc.variables,
themes: modDoc.themes,
})
accumulated = rawResponse
updateLastMessage(accumulated)
@ -180,11 +192,14 @@ function useChatHandlers() {
appliedCount += count
} else {
// --- GENERATION MODE (animated) ---
const doc = useDocumentStore.getState().document
const { rawResponse, nodes } = await generateDesign({
prompt: fullUserMessage,
context: {
canvasSize: { width: 1200, height: 800 },
documentSummary: `Current selection: ${hasSelection ? selectedIds.length + ' items' : 'Empty'}`,
variables: doc.variables,
themes: doc.themes,
},
}, {
animated: true,

View file

@ -125,7 +125,13 @@ DESIGN GUIDELINES:
- Consistent color palette
- Use path nodes for icons (SVG d path data). Size icons 16-24px. Preserve the natural aspect ratio of the SVG path — do NOT force all icons to square
- Use image nodes for photos/illustrations with picsum.photos placeholder URLs
- Buttons, nav items, and list items should include icons when appropriate for better UX`
- Buttons, nav items, and list items should include icons when appropriate for better UX
DESIGN VARIABLES:
- When the user message includes a DOCUMENT VARIABLES section, use "$variableName" references instead of hardcoded values wherever a matching variable exists.
- Color variables: use in fill color, stroke color, shadow color. Example: [{ "type": "solid", "color": "$primary" }]
- Number variables: use for gap, padding, opacity. Example: "gap": "$spacing-md"
- Only reference variables that are listed — do NOT invent new variable names.`
export const DESIGN_GENERATOR_PROMPT = `You are a PenNode JSON generation engine. Your ONLY job is to convert design descriptions into PenNode JSON.
@ -183,6 +189,13 @@ VISUAL QUALITY GUARDRAILS:
- Avoid oversized decorations outside the root frame (max ~10% bleed allowed)
- Do not use emoji in headings or body copy unless the user explicitly asks for it
DESIGN VARIABLES:
- When the user message includes a DOCUMENT VARIABLES section, use "$variableName" references instead of hardcoded values wherever a matching variable exists.
- Color variables: use in fill color, stroke color, shadow color. Example: [{ "type": "solid", "color": "$primary" }]
- Number variables: use for gap, padding, opacity. Example: "gap": "$spacing-md"
- Only reference variables that are listed — do NOT invent new variable names.
- If no variables are provided, use hardcoded values as usual.
Design like a professional: visual hierarchy, contrast, whitespace, consistent palette, purposeful iconography.`
export const CODE_GENERATOR_PROMPT = `You are a code generation engine for OpenPencil. Convert PenNode design descriptions into clean, production-ready code.
@ -214,6 +227,7 @@ RULES:
- PRESERVE IDs: The most important rule. If you return a node with a new ID, it will be treated as a new object. To update, you MUST match the input ID.
- PARTIAL UPDATES: You can return the full node object with updated fields.
- DO NOT CHANGE UNRELATED PROPS: If the user says "change color", do not change the x/y position unless necessary.
- DESIGN VARIABLES: When the user message includes a DOCUMENT VARIABLES section, prefer "$variableName" references over hardcoded values for matching properties. Only reference listed variables.
RESPONSE FORMAT:
1. <step title="Checking guidelines">...</step>

View file

@ -12,6 +12,8 @@ export interface AIDesignRequest {
selectedNodes?: string[]
documentSummary?: string
canvasSize?: { width: number; height: number }
variables?: Record<string, import('@/types/variables').VariableDefinition>
themes?: Record<string, string[]>
}
}

View file

@ -1,4 +1,5 @@
import type { PenNode } from '@/types/pen'
import type { VariableDefinition, ThemedValue } from '@/types/variables'
import type { AIDesignRequest } from './ai-types'
import { streamChat } from './ai-service'
import { DESIGN_GENERATOR_PROMPT, DESIGN_MODIFIER_PROMPT } from './ai-prompts'
@ -78,12 +79,48 @@ function buildContextMessage(request: AIDesignRequest): string {
message += `\n\nCurrent document: ${request.context.documentSummary}`
}
// Append variable context so AI can use $variable references
const varContext = buildVariableContext(request.context?.variables, request.context?.themes)
if (varContext) {
message += `\n\n${varContext}`
}
// FORCE override to prevent tool usage
message += `\n\nIMPORTANT: You remain in DIRECT RESPONSE MODE. Do NOT use the "Write" tool or any other function. I cannot see tool outputs. Just write the JSON response directly.`
return message
}
/** Build a concise summary of document variables for AI context. */
function buildVariableContext(
variables?: Record<string, VariableDefinition>,
themes?: Record<string, string[]>,
): string | null {
if (!variables || Object.keys(variables).length === 0) return null
const lines: string[] = ['DOCUMENT VARIABLES (use "$name" to reference, e.g. fill color "$color-1"):']
for (const [name, def] of Object.entries(variables)) {
const val = def.value
if (Array.isArray(val)) {
// Themed variable — show default value
const defaultVal = (val as ThemedValue[])[0]?.value ?? '?'
lines.push(` - ${name} (${def.type}): ${defaultVal} [themed]`)
} else {
lines.push(` - ${name} (${def.type}): ${val}`)
}
}
if (themes && Object.keys(themes).length > 0) {
const themeSummary = Object.entries(themes)
.map(([axis, values]) => `${axis}: [${values.join(', ')}]`)
.join('; ')
lines.push(`Themes: ${themeSummary}`)
}
return lines.join('\n')
}
/**
* Helper to find all complete JSON blocks in text
*/
@ -259,6 +296,10 @@ function tryParseNodes(json: string): PenNode[] | null {
export async function generateDesignModification(
nodesToModify: PenNode[],
instruction: string,
options?: {
variables?: Record<string, VariableDefinition>
themes?: Record<string, string[]>
},
): Promise<{ nodes: PenNode[]; rawResponse: string }> {
// Build context from selected nodes
const contextJson = JSON.stringify(nodesToModify, (_key, value) => {
@ -267,7 +308,13 @@ export async function generateDesignModification(
})
// We use standard string concatenation to avoid backtick issues in tool calls
const userMessage = "CONTEXT NODES:\n" + contextJson + "\n\nINSTRUCTION:\n" + instruction
let userMessage = "CONTEXT NODES:\n" + contextJson + "\n\nINSTRUCTION:\n" + instruction
// Append variable context so AI can use $variable references
const varContext = buildVariableContext(options?.variables, options?.themes)
if (varContext) {
userMessage += "\n\n" + varContext
}
let fullResponse = ''
let streamError: string | null = null