fix(chat): align vision and model role UX

- Restore attachment inset spacing through the InputGroup theme slot\n- Mark curated multimodal models as Vision-capable and repair saved capability metadata\n- Default Vision to the Design profile when that profile accepts images\n- Use configured Design profiles in the composer instead of raw provider models\n- Surface missing Vision configuration directly in Settings\n\nCo-authored-by: Jason Kneen <jason.kneen@bouncingfish.com>
This commit is contained in:
Danila Poyarkov 2026-09-03 19:58:09 +03:00
parent a8c7fedadb
commit f5db783797
18 changed files with 159 additions and 53 deletions

View file

@ -229,16 +229,37 @@ export const AI_PROVIDERS: AIProviderDef[] = [
defaultModel: 'anthropic/claude-sonnet-4.6',
supportsCustomModel: true,
models: [
{ id: 'anthropic/claude-sonnet-4.6', name: 'Claude Sonnet 4.6', tag: 'Best for design' },
{ id: 'anthropic/claude-opus-4.6', name: 'Claude Opus 4.6', tag: 'Smartest' },
{ id: 'moonshotai/kimi-k2.5', name: 'Kimi K2.5', tag: 'Vision + code' },
{
id: 'anthropic/claude-sonnet-4.6',
name: 'Claude Sonnet 4.6',
tag: 'Best for design',
capabilities: ['tools', 'vision']
},
{
id: 'anthropic/claude-opus-4.6',
name: 'Claude Opus 4.6',
tag: 'Smartest',
capabilities: ['tools', 'vision']
},
{
id: 'moonshotai/kimi-k2.5',
name: 'Kimi K2.5',
tag: 'Vision + code',
capabilities: ['tools', 'vision']
},
{
id: 'google/gemini-3.1-pro-preview',
name: 'Gemini 3.1 Pro',
tag: '1M context'
tag: '1M context',
capabilities: ['tools', 'vision']
},
{ id: 'openai/gpt-5.3-codex', name: 'GPT-5.3 Codex' },
{ id: 'google/gemini-3-flash-preview', name: 'Gemini 3 Flash', tag: 'Fast' },
{
id: 'google/gemini-3-flash-preview',
name: 'Gemini 3 Flash',
tag: 'Fast',
capabilities: ['tools', 'vision']
},
{ id: 'deepseek/deepseek-v3.2', name: 'DeepSeek V3.2', tag: 'Cheap' },
{ id: 'qwen/qwen3.5-flash-02-23', name: 'Qwen 3.5 Flash', tag: 'Cheap' },
{ id: 'qwen/qwen3-coder:free', name: 'Qwen3 Coder', tag: 'Free' },
@ -252,8 +273,18 @@ export const AI_PROVIDERS: AIProviderDef[] = [
keyURL: 'https://console.anthropic.com/settings/keys',
defaultModel: 'claude-sonnet-4-6-20260301',
models: [
{ id: 'claude-sonnet-4-6-20260301', name: 'Claude Sonnet 4.6', tag: 'Best for design' },
{ id: 'claude-opus-4-6-20260301', name: 'Claude Opus 4.6', tag: 'Smartest' }
{
id: 'claude-sonnet-4-6-20260301',
name: 'Claude Sonnet 4.6',
tag: 'Best for design',
capabilities: ['tools', 'vision']
},
{
id: 'claude-opus-4-6-20260301',
name: 'Claude Opus 4.6',
tag: 'Smartest',
capabilities: ['tools', 'vision']
}
]
},
{
@ -276,8 +307,18 @@ export const AI_PROVIDERS: AIProviderDef[] = [
keyURL: 'https://aistudio.google.com/apikey',
defaultModel: 'gemini-3.1-pro-preview',
models: [
{ id: 'gemini-3.1-pro-preview', name: 'Gemini 3.1 Pro', tag: '1M context' },
{ id: 'gemini-3-flash-preview', name: 'Gemini 3 Flash', tag: 'Fast' }
{
id: 'gemini-3.1-pro-preview',
name: 'Gemini 3.1 Pro',
tag: '1M context',
capabilities: ['tools', 'vision']
},
{
id: 'gemini-3-flash-preview',
name: 'Gemini 3 Flash',
tag: 'Fast',
capabilities: ['tools', 'vision']
}
]
},
{

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "Das Anbieterkonto hat nicht genügend Guthaben oder Kontingent. Lade es auf und versuche es erneut.",
"chatOutputLimit": "Das Modell hat beim Schlussfolgern das Ausgabetoken-Limit erreicht. Erhöhe die maximalen Ausgabetoken oder verringere den Schlussfolgerungsaufwand.",
"chatRequestFailed": "Die Modellanfrage ist fehlgeschlagen. Prüfe die Anbietereinstellungen und versuche es erneut.",
"visionModelUnavailable": "Wähle in den Einstellungen ein Vision-Modell aus, bevor du Bilder anhängst.",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "Verbinde einen KI-Anbieter, um den Chat zu starten.",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "La cuenta del proveedor no tiene suficiente saldo o cuota. Recárgala e inténtalo de nuevo.",
"chatOutputLimit": "El modelo agotó los tokens de salida mientras razonaba. Aumenta los tokens máximos de salida o reduce el esfuerzo de razonamiento.",
"chatRequestFailed": "La solicitud al modelo falló. Comprueba la configuración del proveedor e inténtalo de nuevo.",
"visionModelUnavailable": "Elige un modelo de visión en Ajustes antes de adjuntar imágenes.",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "Conecta un proveedor de IA para empezar a chatear.",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "Le compte du fournisseur n’a pas assez de crédit ou de quota. Rechargez-le et réessayez.",
"chatOutputLimit": "Le modèle a épuisé ses jetons de sortie pendant le raisonnement. Augmentez le nombre maximal de jetons de sortie ou réduisez l’effort de raisonnement.",
"chatRequestFailed": "La requête au modèle a échoué. Vérifiez les paramètres du fournisseur et réessayez.",
"visionModelUnavailable": "Choisissez un modèle Vision dans les paramètres avant de joindre des images.",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "Connectez un fournisseur d’IA pour commencer à discuter.",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "L’account del provider non dispone di credito o quota sufficiente. Ricaricalo e riprova.",
"chatOutputLimit": "Il modello ha esaurito i token di output durante il ragionamento. Aumenta i token di output massimi o riduci lo sforzo di ragionamento.",
"chatRequestFailed": "La richiesta al modello non è riuscita. Controlla le impostazioni del provider e riprova.",
"visionModelUnavailable": "Scegli un modello Vision nelle Impostazioni prima di allegare immagini.",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "Collega un provider AI per iniziare a chattare.",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "プロバイダーのアカウントに十分なクレジットまたは割り当てがありません。チャージして再試行してください。",
"chatOutputLimit": "推論中にモデルの出力トークンが上限に達しました。最大出力トークンを増やすか、推論の強度を下げてください。",
"chatRequestFailed": "モデルへのリクエストに失敗しました。プロバイダー設定を確認して再試行してください。",
"visionModelUnavailable": "画像を添付する前に、設定でビジョンモデルを選択してください。",
"completions": "入力補完",
"responses": "応答",
"connectProvider": "チャットを始めるには、AIプロバイダーを接続してください。",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "Konto dostawcy nie ma wystarczających środków lub limitu. Doładuj je i spróbuj ponownie.",
"chatOutputLimit": "Model wyczerpał limit tokenów wyjściowych podczas rozumowania. Zwiększ maksymalną liczbę tokenów wyjściowych lub zmniejsz poziom rozumowania.",
"chatRequestFailed": "Żądanie do modelu nie powiodło się. Sprawdź ustawienia dostawcy i spróbuj ponownie.",
"visionModelUnavailable": "Przed dołączeniem obrazów wybierz model Vision w Ustawieniach.",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "Połącz dostawcę AI, aby rozpocząć czat.",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "На счёте провайдера недостаточно средств или квоты. Пополните его и повторите попытку.",
"chatOutputLimit": "Модель исчерпала лимит выходных токенов во время рассуждения. Увеличьте максимум выходных токенов или снизьте глубину рассуждения.",
"chatRequestFailed": "Запрос к модели завершился ошибкой. Проверьте настройки провайдера и повторите попытку.",
"visionModelUnavailable": "Перед добавлением изображений выберите модель для зрения в настройках.",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "Подключите AI-провайдера, чтобы начать чат.",

View file

@ -91,6 +91,7 @@
"chatInsufficientCredit": "提供商账户的余额或配额不足。请充值后重试。",
"chatOutputLimit": "模型在推理时耗尽了输出 token。请提高最大输出 token 数或降低推理强度。",
"chatRequestFailed": "模型请求失败。请检查提供商设置后重试。",
"visionModelUnavailable": "附加图片前,请在设置中选择视觉模型。",
"completions": "Completions",
"responses": "Responses",
"connectProvider": "连接 AI 提供商以开始聊天。",

View file

@ -106,6 +106,7 @@ export const aiMessageDefaults = {
chatOutputLimit:
'The model ran out of output tokens while reasoning. Raise Max output tokens or lower reasoning effort.',
chatRequestFailed: 'The model request failed. Check the provider settings and try again.',
visionModelUnavailable: 'Choose a Vision model in Settings before attaching images.',
completions: 'Completions',
responses: 'Responses'
} as const

View file

@ -14,6 +14,13 @@ const MAX_IMAGE_ANALYSIS_TOKENS = 1200
export type ImageAnalysisDependencies = VisionModelDependencies
export class VisionModelUnavailableError extends Error {
constructor() {
super('Configure a Vision model in Settings to attach images.')
this.name = 'VisionModelUnavailableError'
}
}
export async function analyzeAttachedImages(
store: EditorStore,
instruction: string,
@ -24,9 +31,7 @@ export async function analyzeAttachedImages(
}
): Promise<string> {
const runtime = await dependencies.createRuntime('vision')
if (runtime?.kind !== 'direct') {
throw new Error('Configure a vision-capable model in Settings to attach images.')
}
if (runtime?.kind !== 'direct') throw new VisionModelUnavailableError()
const content: Array<
| { type: 'text'; text: string }

View file

@ -119,6 +119,29 @@ function optionalAssignment(value: unknown, modelIds: Set<string>): AIModelRoleA
return typeof value === 'string' && modelIds.has(value) ? (value as AIModelProfileId) : null
}
function curatedModelCapabilities(
providerID: AIProviderID,
modelID: string
): AIModelCapability[] | null {
const model = AI_PROVIDERS.find((provider) => provider.id === providerID)?.models.find(
(candidate) => candidate.id === modelID
)
return model?.capabilities ? [...model.capabilities] : null
}
function hydrateCuratedCapabilities(
profiles: AIModelProfile[],
connections: AIModelConnection[]
): void {
for (const profile of profiles) {
if (profile.customModelID) continue
const connection = connections.find((candidate) => candidate.id === profile.connectionId)
if (!connection) continue
const capabilities = curatedModelCapabilities(connection.providerID, profile.modelID)
if (capabilities) profile.capabilities = [...new Set(capabilities)]
}
}
function parseSettings(value: unknown): AIModelSettings | null {
if (!isRecord(value) || value.version !== 1) return null
const connections = Array.isArray(value.connections)
@ -131,6 +154,7 @@ function parseSettings(value: unknown): AIModelSettings | null {
.filter((profile) => profile !== null)
: []
if (!models.length) return null
hydrateCuratedCapabilities(models, connections)
const modelIds = new Set(models.map((profile) => profile.id))
const rawAssignments = isRecord(value.assignments) ? value.assignments : {}
const rawDesign = stringValue(rawAssignments.design)
@ -167,6 +191,7 @@ function legacySettings(): AIModelSettings {
? provider?.models.find((model) => model.id === displayModel)?.name || displayModel
: 'Design model'
const maxOutputTokens = Number(readLegacyAIModelStorage('ai-max-output-tokens'))
const curatedCapabilities = customModelID ? null : curatedModelCapabilities(providerID, modelID)
return {
version: 1,
connections: [
@ -189,14 +214,14 @@ function legacySettings(): AIModelSettings {
maxOutputTokens: Number.isFinite(maxOutputTokens)
? maxOutputTokens
: DEFAULT_MAX_OUTPUT_TOKENS,
capabilities: ['tools']
capabilities: curatedCapabilities ?? ['tools']
}
],
assignments: {
design: LEGACY_MODEL_ID,
review: 'design',
fast: 'design',
vision: null
vision: curatedCapabilities?.includes('vision') ? 'design' : null
}
}
}
@ -372,6 +397,13 @@ export function saveModelProfileDraft(draft: AIModelProfileDraft): AIModelProfil
const index = aiModelSettings.value.models.findIndex((model) => model.id === profile.id)
if (index === -1) aiModelSettings.value.models.push(profile)
else aiModelSettings.value.models[index] = profile
if (
aiModelSettings.value.assignments.vision === null &&
aiModelSettings.value.assignments.design === profile.id &&
profile.capabilities.includes('vision')
) {
aiModelSettings.value.assignments.vision = 'design'
}
if (
aiModelSettings.value.assignments.vision === profile.id &&
!profile.capabilities.includes('vision')

View file

@ -8,7 +8,8 @@ import { copyChatLog } from '@/app/ai/debug'
import { clearVisibleMessageText, setVisibleMessageText } from '@/app/ai/chat/presentation'
import {
analyzeAttachedImages,
designMessageWithImageFindings
designMessageWithImageFindings,
VisionModelUnavailableError
} from '@/app/ai/attachment/image/analyze'
import {
createImagePreviewURL,
@ -32,6 +33,7 @@ import AppButton from '@/components/ui/AppButton.vue'
import ProviderSetup from '@/components/chat/ProviderSetup.vue'
import { useAIChat } from '@/app/ai/chat/use'
import { toast } from '@/app/shell/ui'
import { openSettingsDialog } from '@/app/settings/dialog'
import { useI18n } from '@open-pencil/vue'
import { useNotificationMessages } from '@/app/i18n/notifications'
@ -151,6 +153,16 @@ async function sendTextMessage(
if (message) setVisibleMessageText(message.id, displayText)
}
function reportSubmitError(error: unknown): void {
console.error('Chat error:', error)
if (error instanceof VisionModelUnavailableError) {
toast.error(ai.value.visionModelUnavailable)
openSettingsDialog('ai')
return
}
toast.error(ai.value.chatRequestFailed)
}
async function handleSubmit(text: string, images: ImageAttachmentDraft[] = [], displayText = text) {
if (status.value === 'streaming' || status.value === 'submitted' || isPreparingImages.value) {
for (const image of images) revokeImagePreviewURL(image.previewURL)
@ -229,9 +241,8 @@ async function handleSubmit(text: string, images: ImageAttachmentDraft[] = [], d
findings
)
})
} catch (e) {
console.error('Chat error:', e)
toast.error(ai.value.chatRequestFailed)
} catch (error) {
reportSubmitError(error)
} finally {
if (operationVersion === attachmentOperationVersion) isPreparingImages.value = false
}

View file

@ -7,7 +7,6 @@ import { useI18n, useSelectionState } from '@open-pencil/vue'
import ChatNodePreview from '@/components/chat/ChatNodePreview.vue'
import ChatProfileSelect from '@/components/chat/ChatProfileSelect.vue'
import ProviderModelSelect from '@/components/chat/ProviderModelSelect.vue'
import IconButton from '@/components/ui/IconButton.vue'
import InputGroup from '@/components/ui/InputGroup.vue'
import { useAIChat } from '@/app/ai/chat/use'
@ -16,7 +15,7 @@ import {
MAX_REFERENCED_NODES,
resolveReferencedNodes
} from '@/app/ai/chat/context'
import { designModelProfile, designModelProfiles } from '@/app/ai/models'
import { designModelProfile } from '@/app/ai/models'
import {
createImagePreviewURL,
revokeImagePreviewURL,
@ -125,9 +124,7 @@ const selectedModelName = computed(() => {
return providerDef.value.models.find((m) => m.id === modelID.value)?.name ?? modelID.value
})
// Switching between saved profiles only makes sense once more than one can drive the design agent.
const switchableProfiles = computed(designModelProfiles)
const canSwitchProfile = computed(() => switchableProfiles.value.length > 1)
// The composer switches configured Design-role profiles; raw provider model selection lives in Settings.
const selectedProfileName = computed(
() => designModelProfile.value?.name ?? selectedModelName.value
)
@ -284,26 +281,11 @@ function handleSubmit(e: Event) {
<span class="truncate">{{ agentName }}</span>
</div>
</template>
<ChatProfileSelect
v-else-if="canSwitchProfile && (isCustomProvider || usesCustomModel)"
>
<ChatProfileSelect v-else>
<template #value>
<span class="min-w-0 truncate">{{ selectedProfileName }}</span>
</template>
</ChatProfileSelect>
<div
v-else-if="isCustomProvider || usesCustomModel"
class="flex min-w-0 items-center gap-1 px-1.5 text-[10px] text-muted"
data-test-id="chat-custom-model-label"
>
<icon-lucide-bot class="size-3 shrink-0" />
<span class="truncate">{{ selectedModelName }}</span>
</div>
<ProviderModelSelect v-else>
<template #value>
<span class="min-w-0 truncate">{{ selectedModelName }}</span>
</template>
</ProviderModelSelect>
</div>
</template>

View file

@ -10,9 +10,15 @@ import {
SelectViewport
} from 'reka-ui'
import { AI_PROVIDERS } from '@open-pencil/core/constants'
import { useI18n } from '@open-pencil/vue'
import { aiModelSettings, designModelProfiles, setModelRoleAssignment } from '@/app/ai/models'
import {
aiModelSettings,
designModelProfiles,
modelConnection,
setModelRoleAssignment
} from '@/app/ai/models'
import type { AIModelProfileId } from '@/app/ai/models'
import AppBadge from '@/components/ui/AppBadge.vue'
import { useSelectUI } from '@/components/ui/select'
@ -27,6 +33,13 @@ const selectedProfileId = computed({
set: (profileId: AIModelProfileId) => setModelRoleAssignment('design', profileId)
})
function providerName(connectionId: string): string {
const providerID = modelConnection(connectionId)?.providerID
return providerID
? (AI_PROVIDERS.find((provider) => provider.id === providerID)?.name ?? providerID)
: ''
}
const selectCls = useSelectUI({
trigger:
'min-w-0 max-w-full gap-1 rounded border-none bg-transparent px-1.5 py-0.5 text-[10px] text-muted',
@ -49,13 +62,22 @@ const selectCls = useSelectUI({
<SelectPortal>
<SelectContent position="popper" side="top" :side-offset="4" :class="selectCls.content">
<SelectViewport>
<div class="px-2 pt-1.5 pb-1 text-[9px] font-medium tracking-wide text-muted uppercase">
{{ ai.modelRoleDesign }}
</div>
<SelectItem
v-for="profile in profiles"
:key="profile.id"
:value="profile.id"
:class="selectCls.item"
>
<SelectItemText class="min-w-0 flex-1 truncate">{{ profile.name }}</SelectItemText>
<SelectItemText class="min-w-0 flex-1">
<span class="block truncate">{{ profile.name }}</span>
<span class="block truncate text-[9px] text-muted">
{{ providerName(profile.connectionId) }}
</span>
</SelectItemText>
<AppBadge>{{ ai.modelCapabilityToolsShort }}</AppBadge>
<AppBadge v-if="profile.capabilities.includes('vision')">
{{ ai.modelCapabilityVisionShort }}
</AppBadge>

View file

@ -30,7 +30,11 @@ const cls = computed(() => inputGroup({ size, disabled }))
:data-disabled="disabled || undefined"
:class="cls.root({ class: ui?.root })"
>
<div v-if="$slots.attachment" data-slot="input-group-attachment" :class="ui?.attachment">
<div
v-if="$slots.attachment"
data-slot="input-group-attachment"
:class="cls.attachment({ class: ui?.attachment })"
>
<slot name="attachment" />
</div>
<div data-slot="input-group-control" :class="cls.control({ class: ui?.control })">

View file

@ -3,7 +3,7 @@ import { controlHeight } from '@/theme/control'
export default {
slots: {
root: 'min-w-0 rounded-xl border border-border bg-input transition-colors hover:border-muted/60 focus-within:border-panel-focus focus-within:ring-1 focus-within:ring-accent/30',
attachment: 'px-2 pt-2',
attachment: 'm-2 mb-0',
control: 'min-w-0',
toolbar: 'flex min-w-0 items-center gap-1 px-1.5 pb-1.5',
model: 'min-w-0 flex-1',

View file

@ -410,17 +410,18 @@ test('assistant Markdown restricts images and blocks unsafe link protocols', asy
)
})
test('model selector is visible and clickable', async () => {
const trigger = page.getByTestId('chat-model-selector')
test('design profile selector is visible and exposes role capabilities', async () => {
const trigger = page.getByTestId('chat-profile-selector')
await expect(trigger).toBeVisible()
await trigger.click()
await expect(page.getByRole('option', { name: /Claude Sonnet 4\.6/ })).toBeVisible()
await expect(page.getByText('Best for design')).toBeVisible()
await expect(page.getByText('Free').first()).toBeVisible()
await expect(page.getByText('Design agent', { exact: true })).toBeVisible()
await expect(page.getByRole('option', { name: /Claude Sonnet/ })).toContainText('OpenRouter')
await expect(page.getByRole('option', { name: /Claude Sonnet/ })).toContainText('Tools')
await expect(page.getByRole('option', { name: /Claude Sonnet/ })).toContainText('Vision')
await page.getByRole('option', { name: /Claude Sonnet 4\.6/ }).click()
await expect(page.getByRole('option', { name: /Claude Sonnet 4\.6/ })).toBeHidden()
await page.getByRole('option', { name: /Claude Sonnet/ }).click()
await expect(page.getByRole('option', { name: /Claude Sonnet/ })).toBeHidden()
})
test('reasoning and response copy actions render in assistant messages', async () => {
@ -490,8 +491,7 @@ test('OpenRouter accepts a custom model ID from provider settings', async () =>
await page.getByRole('button', { name: 'Save model' }).click()
await page.getByTestId('app-settings-done').click()
await expect(page.getByTestId('chat-custom-model-label')).toContainText(customModel)
await expect(page.getByTestId('chat-model-selector')).toBeHidden()
await expect(page.getByTestId('chat-profile-selector')).toContainText('Claude Sonnet')
await page.getByTestId('provider-settings-trigger').click()
await page.locator('[data-model-id]').first().click()
@ -502,7 +502,7 @@ test('OpenRouter accepts a custom model ID from provider settings', async () =>
await page.getByRole('button', { name: 'Save model' }).click()
await page.getByTestId('app-settings-done').click()
await expect(page.getByTestId('chat-model-selector')).toBeVisible()
await expect(page.getByTestId('chat-profile-selector')).toBeVisible()
})
test('transport errors show a safe localized toast', async () => {