feat(canvas): paste clipboard images as nodes

This commit is contained in:
Kayshen-X 2026-07-12 09:07:49 +08:00
parent 4fb6510ce0
commit 3a825cbfe1
7 changed files with 507 additions and 33 deletions

View file

@ -214,11 +214,41 @@ impl EditorState {
/// Returns the new node id on success; `None` when the id allocator is
/// exhausted.
pub fn insert_image_node_at_viewport(&mut self, name: &str, src: &str) -> Option<NodeId> {
self.insert_image_node_at_viewport_with_dimensions(name, src, 300.0, 200.0)
}
/// Insert an Image node centred on the current viewport, preserving the
/// source bitmap's aspect ratio. The largest side is capped at 300
/// document pixels and smaller bitmaps are not enlarged.
pub fn insert_image_node_at_viewport_sized(
&mut self,
name: &str,
src: &str,
pixel_width: u32,
pixel_height: u32,
) -> Option<NodeId> {
if pixel_width == 0 || pixel_height == 0 {
return None;
}
let scale = (300.0 / f64::from(pixel_width.max(pixel_height))).min(1.0);
self.insert_image_node_at_viewport_with_dimensions(
name,
src,
f64::from(pixel_width) * scale,
f64::from(pixel_height) * scale,
)
}
fn insert_image_node_at_viewport_with_dimensions(
&mut self,
name: &str,
src: &str,
width: f64,
height: f64,
) -> Option<NodeId> {
use jian_ops_schema::node::image::ImageNode;
use jian_ops_schema::node::PenNode;
use jian_ops_schema::sizing::SizingBehavior;
const W: f64 = 300.0;
const H: f64 = 200.0;
let pan_x = self.viewport.pan_x as f64;
let pan_y = self.viewport.pan_y as f64;
let zoom = self.viewport.zoom.max(0.001) as f64;
@ -226,21 +256,20 @@ impl EditorState {
let centre_y = -pan_y / zoom;
let safe = self.max_node_id().checked_add(1)?;
let id = NodeId::new(format!("n{}", safe));
let mut next_id = safe.checked_add(1)?;
let _ = &mut next_id;
let _next_id = safe.checked_add(1)?;
self.commit_history();
let node = PenNode::Image(ImageNode {
base: jian_ops_schema::node::base::PenNodeBase {
id: id.as_str().to_string(),
name: Some(name.to_string()),
x: Some(centre_x - W / 2.0),
y: Some(centre_y - H / 2.0),
x: Some(centre_x - width / 2.0),
y: Some(centre_y - height / 2.0),
..Default::default()
},
src: src.into(),
object_fit: None,
width: Some(SizingBehavior::Number(W)),
height: Some(SizingBehavior::Number(H)),
width: Some(SizingBehavior::Number(width)),
height: Some(SizingBehavior::Number(height)),
corner_radius: None,
effects: None,
exposure: None,

View file

@ -0,0 +1,86 @@
use crate::{EditorState, NodeId, Viewport};
use jian_ops_schema::node::PenNode;
use jian_ops_schema::sizing::SizingBehavior;
#[test]
fn sized_image_preserves_aspect_ratio_centers_selects_and_undoes() {
let mut state = EditorState::sample();
state.viewport = Viewport {
pan_x: -200.0,
pan_y: -100.0,
zoom: 2.0,
};
let id = state
.insert_image_node_at_viewport_sized(
"Clipboard image",
"data:image/png;base64,AAAA",
400,
200,
)
.expect("image inserted");
assert_eq!(id, NodeId::new("n15"));
assert_eq!(state.selection.anchor, id);
assert_eq!(state.active_children().len(), 2);
let PenNode::Image(image) = &state.active_children()[0] else {
panic!("inserted node should be an Image");
};
assert_eq!(image.base.name.as_deref(), Some("Clipboard image"));
assert_eq!(image.base.x, Some(-50.0));
assert_eq!(image.base.y, Some(-25.0));
assert_eq!(image.width, Some(SizingBehavior::Number(300.0)));
assert_eq!(image.height, Some(SizingBehavior::Number(150.0)));
assert_eq!(image.src, "data:image/png;base64,AAAA");
assert!(state.undo());
assert_eq!(state.active_children().len(), 1);
assert_eq!(state.selection.anchor, NodeId::new("n11"));
}
#[test]
fn sized_image_rejects_zero_pixel_dimensions_without_history() {
for (pixel_width, pixel_height) in [(0, 100), (100, 0), (0, 0)] {
let mut state = EditorState::sample();
assert!(state
.insert_image_node_at_viewport_sized(
"Invalid image",
"data:image/png;base64,AAAA",
pixel_width,
pixel_height,
)
.is_none());
assert_eq!(state.active_children().len(), 1);
assert_eq!(state.selection.anchor, NodeId::new("n11"));
assert!(!state.history.can_undo());
}
}
#[test]
fn sized_image_keeps_small_bitmap_at_original_size_and_centers_it() {
let mut state = EditorState::new();
state.viewport = Viewport {
pan_x: -200.0,
pan_y: -100.0,
zoom: 2.0,
};
let id = state
.insert_image_node_at_viewport_sized(
"Small clipboard image",
"data:image/png;base64,AAAA",
64,
32,
)
.expect("small image inserted");
let PenNode::Image(image) = state.selected_node().expect("inserted image selected") else {
panic!("inserted node should be an Image");
};
assert_eq!(state.selection.anchor, id);
assert_eq!(image.width, Some(SizingBehavior::Number(64.0)));
assert_eq!(image.height, Some(SizingBehavior::Number(32.0)));
assert_eq!(image.base.x, Some(68.0));
assert_eq!(image.base.y, Some(34.0));
}

View file

@ -55,6 +55,8 @@ pub mod history;
pub mod history_snapshot;
pub mod hoist_app_state;
pub mod host_support;
#[cfg(test)]
mod host_support_tests;
pub mod icon_picker_state;
pub mod image_node_props;
pub mod image_panel_state;

View file

@ -6,8 +6,17 @@
//! a clipboard that fails to initialise simply yields `None` / a
//! no-op rather than surfacing an error to the user.
/// PNG-encoded clipboard bitmap plus its original pixel dimensions.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ClipboardImage {
pub png: Vec<u8>,
pub width: u32,
pub height: u32,
}
/// Read the system clipboard's text. `None` when the clipboard holds
/// no text or could not be opened.
#[allow(dead_code)] // Retained single-flavour API; paste uses one shared OS handle.
pub fn get_text() -> Option<String> {
arboard::Clipboard::new().ok()?.get_text().ok()
}
@ -18,8 +27,26 @@ pub fn get_text() -> Option<String> {
/// wrapped / encoded. Backs paste-image-into-chat — the desktop
/// equivalent of the TS chat input's clipboard *files* surface
/// (`ai-chat-input.tsx:85-94`).
pub fn get_image() -> Option<Vec<u8>> {
#[allow(dead_code)] // Required typed single-image API; paste uses the snapshot API below.
pub fn get_image() -> Option<ClipboardImage> {
let image = arboard::Clipboard::new().ok()?.get_image().ok()?;
encode_image(image)
}
/// Snapshot all paste-relevant clipboard flavours through one OS
/// clipboard handle. The keyboard router turns this tuple into its
/// injectable `ClipboardPayload` seam.
pub(crate) fn read_paste_flavours() -> (Option<String>, Option<String>, Option<ClipboardImage>) {
let Ok(mut clipboard) = arboard::Clipboard::new() else {
return (None, None, None);
};
let text = clipboard.get_text().ok();
let html = clipboard.get().html().ok();
let image = clipboard.get_image().ok().and_then(encode_image);
(text, html, image)
}
fn encode_image(image: arboard::ImageData<'_>) -> Option<ClipboardImage> {
let (width, height) = (image.width, image.height);
if width == 0 || height == 0 {
return None;
@ -27,25 +54,34 @@ pub fn get_image() -> Option<Vec<u8>> {
// arboard hands back tightly-packed RGBA8; wrap it as a raster
// skia image and run it through the same PNG encoder the export
// path uses (no extra image-codec dependency).
if image.bytes.len() < width * height * 4 {
let row_bytes = width.checked_mul(4)?;
let rgba_len = row_bytes.checked_mul(height)?;
if image.bytes.len() < rgba_len {
return None;
}
let pixel_width = u32::try_from(width).ok()?;
let pixel_height = u32::try_from(height).ok()?;
let info = skia_safe::ImageInfo::new(
(width as i32, height as i32),
(i32::try_from(width).ok()?, i32::try_from(height).ok()?),
skia_safe::ColorType::RGBA8888,
skia_safe::AlphaType::Unpremul,
None,
);
let data = skia_safe::Data::new_copy(&image.bytes);
let raster = skia_safe::images::raster_from_data(&info, data, width * 4)?;
let raster = skia_safe::images::raster_from_data(&info, data, row_bytes)?;
let png = raster.encode(None, skia_safe::EncodedImageFormat::PNG, 100)?;
Some(png.as_bytes().to_vec())
Some(ClipboardImage {
png: png.as_bytes().to_vec(),
width: pixel_width,
height: pixel_height,
})
}
/// Read the system clipboard's HTML flavour (NSPasteboard
/// `public.html` / CF_HTML / text/html). `None` when the clipboard
/// holds no HTML — the Figma-paste path probes this before falling
/// back to the internal node clipboard.
#[allow(dead_code)] // Retained single-flavour API; paste uses one shared OS handle.
pub fn get_html() -> Option<String> {
arboard::Clipboard::new().ok()?.get().html().ok()
}

View file

@ -3,8 +3,26 @@
//! under the repo's 800-line-per-file cap.
use crate::{chat_session, persistence, DesktopApp};
use base64::Engine as _;
use winit::keyboard::{Key, NamedKey};
/// Snapshot of every system-clipboard flavour relevant to Cmd/Ctrl+V.
/// Tests inject this directly so paste precedence is deterministic and
/// never depends on the developer machine's clipboard contents.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct ClipboardPayload {
pub(crate) text: Option<String>,
pub(crate) html: Option<String>,
pub(crate) image: Option<crate::clipboard::ClipboardImage>,
}
impl ClipboardPayload {
fn read_system() -> Self {
let (text, html, image) = crate::clipboard::read_paste_flavours();
Self { text, html, image }
}
}
impl DesktopApp {
/// Dispatch a pressed key (`logical_key` + its `text`) — the
/// editor's keyboard-shortcut table. Called from the winit
@ -382,28 +400,40 @@ impl DesktopApp {
/// pastes Figma clipboard HTML or the document node clipboard onto
/// the canvas. `pub(crate)` for the Edit-menu path.
pub(crate) fn handle_cmd_paste(&mut self) -> bool {
self.handle_paste_payload(ClipboardPayload::read_system())
}
/// Input-aware paste router over an already-read clipboard snapshot.
/// Keeping OS access outside this seam makes precedence directly
/// testable and guarantees one clipboard handle per paste gesture.
pub(crate) fn handle_paste_payload(&mut self, mut payload: ClipboardPayload) -> bool {
// Non-chat text inputs (settings, git, rename, etc.) are
// checked first so that opening a modal (e.g. agent settings
// via Cmd+,) while the chat input is focused routes the paste
// to the modal's field instead of the chat.
if self.host.input_active_pub() && !self.host.editor_state().chat.focused {
if let Some(text) = crate::clipboard::get_text() {
self.host.apply_input_paste(&text);
if self.host.non_chat_input_owns_keyboard_pub() {
if let Some(text) = payload.text.as_deref() {
self.host.apply_input_paste(text);
}
return true;
}
if self.host.editor_state().chat.focused {
if self.host.chat_input_owns_keyboard_pub() {
// Clipboard image data wins over text when pasting into the
// chat input; the paste is consumed either way.
if !self.try_paste_image_into_chat() {
if let Some(text) = crate::clipboard::get_text() {
self.host.chat_input_paste(&text);
}
if let Some(image) = payload.image.take() {
self.paste_image_into_chat(image);
} else if let Some(text) = payload.text.as_deref() {
self.host.chat_input_paste(text);
}
return true;
}
if let Some(result) = self.try_figma_clipboard_paste() {
return result;
if let Some(html) = payload.html.take() {
if let Some(result) = self.try_figma_clipboard_paste(html) {
return result;
}
}
if let Some(image) = payload.image {
return self.paste_image_to_canvas(image);
}
self.host.apply_paste()
}
@ -414,12 +444,7 @@ impl DesktopApp {
/// and the paste is consumed even when nothing stages — TS
/// filters oversized files after `preventDefault()`, and
/// `add_attachment` enforces the same 4 × 5 MB caps here.
/// Returns false when the clipboard holds no image (caller falls
/// through to the text paste).
fn try_paste_image_into_chat(&mut self) -> bool {
let Some(png) = crate::clipboard::get_image() else {
return false;
};
fn paste_image_into_chat(&mut self, image: crate::clipboard::ClipboardImage) {
// TS names pasted clipboard images "pasted-image.png".
self.host
.editor_state_mut()
@ -427,9 +452,27 @@ impl DesktopApp {
.add_attachment(op_editor_core::chat::ChatAttachment {
name: "pasted-image.png".to_string(),
media_type: "image/png".to_string(),
data: png,
data: image.png,
});
true
}
fn paste_image_to_canvas(&mut self, image: crate::clipboard::ClipboardImage) -> bool {
let encoded = base64::engine::general_purpose::STANDARD.encode(&image.png);
let src = format!("data:image/png;base64,{encoded}");
let inserted = self
.host
.editor_state_mut()
.insert_image_node_at_viewport_sized(
"pasted-image.png",
&src,
image.width,
image.height,
)
.is_some();
if inserted {
self.host.mark_editor_state_dirty();
}
inserted
}
/// Probe the system clipboard for Figma HTML (Cmd+C in Figma) and
@ -439,8 +482,7 @@ impl DesktopApp {
/// marker; `pump_figma_clipboard_paste` applies the parsed nodes
/// on a later frame. `None` when the clipboard holds no Figma
/// payload (caller falls back to the internal node clipboard).
fn try_figma_clipboard_paste(&mut self) -> Option<bool> {
let html = crate::clipboard::get_html()?;
fn try_figma_clipboard_paste(&mut self, html: String) -> Option<bool> {
if !op_figma::is_figma_clipboard_html(&html) {
return None;
}
@ -590,3 +632,7 @@ impl DesktopApp {
self.host.mark_editor_state_dirty();
}
}
#[cfg(test)]
#[path = "keyboard_input_tests.rs"]
mod tests;

View file

@ -0,0 +1,233 @@
use super::*;
use crate::clipboard::ClipboardImage;
use crate::keyboard_input::ClipboardPayload;
use jian_ops_schema::node::PenNode;
use jian_ops_schema::sizing::SizingBehavior;
use op_editor_core::agent_settings::{AcpAgentField, SettingsFocus};
const FIGMA_HTML: &str =
"<html><!--(figmeta)-->eyJ2IjoxfQ==<!--(figmeta)--><!--(figma)-->T1A=<!--(figma)--></html>";
fn image(width: u32, height: u32) -> ClipboardImage {
ClipboardImage {
png: vec![1, 2, 3],
width,
height,
}
}
fn payload(
text: Option<&str>,
html: Option<&str>,
image: Option<ClipboardImage>,
) -> ClipboardPayload {
ClipboardPayload {
text: text.map(str::to_string),
html: html.map(str::to_string),
image,
}
}
fn seed_internal_clipboard(app: &mut DesktopApp) {
let state = app.host.editor_state_mut();
state.set_single_selection(op_editor_core::NodeId::new("n10"));
assert!(state.copy_selected());
state.clear_selection();
}
fn focus_settings_input(app: &mut DesktopApp) {
let ui = &mut app.host.editor_state_mut().editor_ui;
ui.agent_settings_open = true;
ui.agent_settings.add_acp_agent();
ui.agent_settings.focus = Some(SettingsFocus::AcpAgent {
index: 0,
field: AcpAgentField::Command,
});
ui.settings_input.set_text("");
}
#[test]
fn canvas_image_paste_preserves_size_selects_and_beats_internal_nodes() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
app.host.editor_state_mut().viewport = op_editor_core::Viewport::IDENTITY;
assert!(app.handle_paste_payload(payload(None, None, Some(image(400, 200)))));
let state = app.host.editor_state();
assert_eq!(state.active_children().len(), 2);
let id = state.selection.anchor.clone();
let PenNode::Image(node) = op_editor_core::walkers::find_node(state.active_children(), &id)
.expect("selected pasted node exists")
else {
panic!("selected pasted node should be an Image");
};
assert_eq!(node.width, Some(SizingBehavior::Number(300.0)));
assert_eq!(node.height, Some(SizingBehavior::Number(150.0)));
assert_eq!(node.base.x, Some(-150.0));
assert_eq!(node.base.y, Some(-75.0));
assert_eq!(node.src, "data:image/png;base64,AQID");
assert!(state.history.can_undo());
assert!(app.host.editor_state_mut().undo());
assert_eq!(app.host.editor_state().active_children().len(), 1);
}
#[test]
fn focused_non_chat_input_pastes_text_before_every_canvas_flavour() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
focus_settings_input(&mut app);
assert!(app.handle_paste_payload(payload(
Some("codex\n"),
Some(FIGMA_HTML),
Some(image(400, 200)),
)));
assert_eq!(
app.host.editor_state().editor_ui.settings_input.text(),
"codex"
);
assert_eq!(app.host.editor_state().active_children().len(), 1);
assert!(app.host.editor_state().chat.pending_attachments.is_empty());
assert!(app.pending_figma_paste.is_none());
}
#[test]
fn focused_chat_prefers_image_attachment_then_falls_back_to_text() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
app.host.editor_state_mut().chat.focused = true;
assert!(app.handle_paste_payload(payload(
Some("ignored text"),
Some(FIGMA_HTML),
Some(image(640, 480)),
)));
let state = app.host.editor_state();
assert!(state.chat.input.text().is_empty());
assert_eq!(state.chat.pending_attachments.len(), 1);
assert_eq!(state.chat.pending_attachments[0].data, vec![1, 2, 3]);
assert_eq!(state.active_children().len(), 1);
assert!(app.pending_figma_paste.is_none());
app.host.editor_state_mut().chat.pending_attachments.clear();
assert!(app.handle_paste_payload(payload(Some("chat text"), None, None)));
assert_eq!(app.host.editor_state().chat.input.text(), "chat text");
assert!(app.host.editor_state().chat.pending_attachments.is_empty());
}
#[test]
fn chat_model_picker_paste_owns_keyboard_over_stale_chat_focus() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
let state = app.host.editor_state_mut();
state.editor_ui.chat_model_picker.open = true;
state.chat.focused = true;
assert!(app.handle_paste_payload(payload(Some("gp"), Some(FIGMA_HTML), Some(image(640, 480)),)));
let state = app.host.editor_state();
assert_eq!(state.editor_ui.chat_model_picker_input.text(), "gp");
assert!(state.chat.input.text().is_empty());
assert!(state.chat.pending_attachments.is_empty());
assert_eq!(state.active_children().len(), 1);
assert!(state.selection.is_empty());
assert_eq!(state.clipboard.len(), 1);
assert!(app.pending_figma_paste.is_none());
}
#[test]
fn font_picker_paste_owns_keyboard_over_stale_chat_focus() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
let state = app.host.editor_state_mut();
state.editor_ui.font_picker.open = true;
state.chat.focused = true;
assert!(app.handle_paste_payload(payload(
Some("Inter"),
Some(FIGMA_HTML),
Some(image(640, 480)),
)));
let state = app.host.editor_state();
assert_eq!(state.editor_ui.font_picker_search, "Inter");
assert!(state.chat.input.text().is_empty());
assert!(state.chat.pending_attachments.is_empty());
assert_eq!(state.active_children().len(), 1);
assert!(state.selection.is_empty());
assert_eq!(state.clipboard.len(), 1);
assert!(app.pending_figma_paste.is_none());
}
#[test]
fn image_search_paste_owns_keyboard_over_stale_chat_focus() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
let state = app.host.editor_state_mut();
state.editor_ui.image_panel.search_open = true;
state.chat.focused = true;
assert!(app.handle_paste_payload(payload(
Some("sunset"),
Some(FIGMA_HTML),
Some(image(640, 480)),
)));
let state = app.host.editor_state();
assert_eq!(state.editor_ui.image_panel.search_query, "sunset");
assert!(state.chat.input.text().is_empty());
assert!(state.chat.pending_attachments.is_empty());
assert_eq!(state.active_children().len(), 1);
assert!(state.selection.is_empty());
assert_eq!(state.clipboard.len(), 1);
assert!(app.pending_figma_paste.is_none());
}
#[test]
fn hidden_git_focus_does_not_swallow_canvas_image_paste() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
app.host.editor_state_mut().viewport = op_editor_core::Viewport::IDENTITY;
let git = &mut app.host.editor_state_mut().editor_ui.git_panel;
assert!(!git.open);
git.commit_focused = true;
assert!(app.handle_paste_payload(payload(Some("stale git text"), None, Some(image(400, 200)),)));
let state = app.host.editor_state();
assert!(state.editor_ui.git_panel.commit_input.text().is_empty());
assert_eq!(state.active_children().len(), 2);
assert!(matches!(state.selected_node(), Some(PenNode::Image(_))));
assert_eq!(state.clipboard.len(), 1);
}
#[test]
fn figma_html_beats_canvas_image_and_internal_nodes() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
assert!(app.handle_paste_payload(payload(None, Some(FIGMA_HTML), Some(image(400, 200)),)));
assert!(app.pending_figma_paste.is_some());
assert_eq!(app.host.editor_state().active_children().len(), 1);
assert!(app.host.editor_state().selection.is_empty());
}
#[test]
fn internal_node_clipboard_remains_the_canvas_fallback() {
let mut app = DesktopApp::new(None);
seed_internal_clipboard(&mut app);
assert!(app.handle_paste_payload(ClipboardPayload::default()));
assert_eq!(app.host.editor_state().active_children().len(), 2);
assert!(app.host.editor_state().selection.anchor.is_real());
assert!(!matches!(
app.host.editor_state().selected_node(),
Some(PenNode::Image(_))
));
}

View file

@ -5,6 +5,48 @@ use super::WidgetHostNative;
use op_editor_core::ReorderDirection;
impl WidgetHostNative {
/// True when a non-chat text surface owns keyboard input. Plain-string
/// host inputs are checked explicitly; `TextInputState` owners share the
/// canonical editor-core resolver.
pub fn non_chat_input_owns_keyboard_pub(&self) -> bool {
let editor_ui = &self.editor_state.editor_ui;
if self.preview.is_some()
|| editor_ui.font_picker.open
|| editor_ui.image_panel.search_open
|| editor_ui.image_panel.generate_open
|| self.variables_search_active()
|| editor_ui.preset_name_input_active()
|| editor_ui.icon_picker.open
|| editor_ui.component_browser_open
// `active_text_input()` resolves chat before Git. Visible Git /
// clone inputs must therefore claim ownership before the pointer
// comparison below when a stale chat-focus bit coexists.
|| self.git_commit_focus_active()
|| self.git_remote_focus_active()
|| self.git_https_focus_active()
|| self.git_branch_create_focus_active()
|| self.git_author_focus_active()
|| self.git_clone_input_active()
{
return true;
}
if self.editor_state.chat.focused {
self.editor_state
.active_text_input()
.is_some_and(|active| !std::ptr::eq(active, &self.editor_state.chat.input))
} else {
// The host predicate applies the visibility gates for Git /
// clone inputs; the plain-string omissions are covered above.
self.input_active()
}
}
/// True only when the chat input, rather than a higher-priority
/// non-chat surface, owns keyboard input.
pub fn chat_input_owns_keyboard_pub(&self) -> bool {
self.editor_state.chat.focused && !self.non_chat_input_owns_keyboard_pub()
}
/// Cmd-C — copy selection to clipboard.
pub fn apply_copy(&mut self) -> bool {
if self.editor_state.chat.focused {