From b849766cc29df755111fee27d48e7a1401ac6d13 Mon Sep 17 00:00:00 2001 From: Fini Date: Sat, 30 May 2026 07:26:13 +0800 Subject: [PATCH] fix(ai): fill placeholder frames from image search --- .../src/image_search_session.rs | 397 ++++++++++++------ .../src/image_search_session_tests.rs | 241 +++++++++++ 2 files changed, 517 insertions(+), 121 deletions(-) create mode 100644 crates/op-host-desktop/src/image_search_session_tests.rs diff --git a/crates/op-host-desktop/src/image_search_session.rs b/crates/op-host-desktop/src/image_search_session.rs index 8a23b1e4f..38a7d130b 100644 --- a/crates/op-host-desktop/src/image_search_session.rs +++ b/crates/op-host-desktop/src/image_search_session.rs @@ -4,7 +4,8 @@ use std::collections::HashSet; use std::sync::mpsc::{self, Receiver, TryRecvError}; use std::time::Duration; -use jian_ops_schema::node::PenNode; +use jian_ops_schema::node::{PenNode, TextContent}; +use jian_ops_schema::style::{ImageFillBody, ImageFillMode, PenFill}; use op_editor_core::{walkers, EditorState, NodeId, PenNodeExt as _}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -127,7 +128,7 @@ pub(crate) fn collect_targets( known_node_ids: &HashSet, ) -> Vec { let mut targets = Vec::new(); - collect_from_children(state.active_children(), known_node_ids, &mut targets); + collect_from_children(state.active_children(), known_node_ids, &mut targets, &[]); targets } @@ -135,30 +136,256 @@ fn collect_from_children( children: &[PenNode], known_node_ids: &HashSet, targets: &mut Vec, + parent_names: &[String], ) { for node in children { - if let PenNode::Image(image) = node { - let id = image.base.id.as_str(); - let query = image - .image_search_query - .as_deref() - .filter(|q| !q.trim().is_empty()) - .or(image.base.name.as_deref()) - .unwrap_or("placeholder") - .trim(); - if image.src.trim().is_empty() && !query.is_empty() && !known_node_ids.contains(id) { - targets.push(ImageSearchTarget { - node_id: NodeId::new(id), - query: query.to_string(), - }); - } + if let Some(target) = image_search_target_for(node, known_node_ids, parent_names) { + targets.push(target); + } + + if is_image_placeholder_frame(node) || is_image_area_frame_by_heuristic(node) { + continue; } if let Some(grand) = node.children() { - collect_from_children(grand, known_node_ids, targets); + let mut child_parent_names = Vec::with_capacity(parent_names.len() + 1); + child_parent_names.push(node.base().name.clone().unwrap_or_default()); + child_parent_names.extend(parent_names.iter().cloned()); + collect_from_children(grand, known_node_ids, targets, &child_parent_names); } } } +fn image_search_target_for( + node: &PenNode, + known_node_ids: &HashSet, + parent_names: &[String], +) -> Option { + let id = node.base().id.as_str(); + if known_node_ids.contains(id) { + return None; + } + + let needs_image = match node { + PenNode::Image(image) => is_placeholder_src(&image.src), + PenNode::Frame(_) => is_frame_placeholder_still_unfilled(node), + _ => false, + }; + if !needs_image { + return None; + } + + let query = extract_query_for_node(node, parent_names); + if query.is_empty() { + return None; + } + + Some(ImageSearchTarget { + node_id: NodeId::new(id), + query, + }) +} + +fn is_placeholder_src(src: &str) -> bool { + src.trim().is_empty() || src.starts_with("data:image/svg+xml;charset=utf-8,%3Csvg") +} + +fn is_image_placeholder_frame(node: &PenNode) -> bool { + matches!(node, PenNode::Frame(_)) && node.base().role.as_deref() == Some("image-placeholder") +} + +fn is_frame_placeholder_still_unfilled(node: &PenNode) -> bool { + is_unfilled_image_placeholder_frame(node) || is_image_area_frame_by_heuristic(node) +} + +fn is_unfilled_image_placeholder_frame(node: &PenNode) -> bool { + if !is_image_placeholder_frame(node) { + return false; + } + let PenNode::Frame(frame) = node else { + return false; + }; + match frame.container.fill.as_deref() { + None | Some([]) => true, + Some([PenFill::Image(_), ..]) => false, + Some(_) => true, + } +} + +fn is_image_area_frame_by_heuristic(node: &PenNode) -> bool { + let PenNode::Frame(frame) = node else { + return false; + }; + if frame.base.role.as_deref() == Some("image-placeholder") { + return false; + } + let Some(name) = frame.base.name.as_deref() else { + return false; + }; + if !has_image_area_keyword(name) { + return false; + } + if !matches!(node.width_px(), Some(w) if w >= 80.0) { + return false; + } + if !matches!(node.height_px(), Some(h) if h >= 60.0) { + return false; + } + if !matches!(frame.container.fill.as_deref(), Some([PenFill::Solid(_)])) { + return false; + } + let Some(children) = frame.children.as_ref() else { + return true; + }; + match children.as_slice() { + [] => true, + [PenNode::IconFont(_)] => true, + _ => false, + } +} + +fn has_image_area_keyword(name: &str) -> bool { + name.split(|c: char| !c.is_ascii_alphanumeric()) + .map(str::to_ascii_lowercase) + .any(|word| { + matches!( + word.as_str(), + "image" + | "photo" + | "cover" + | "hero" + | "thumbnail" + | "thumb" + | "picture" + | "banner" + | "poster" + ) + }) +} + +fn extract_query_for_node(node: &PenNode, parent_names: &[String]) -> String { + if let PenNode::Image(image) = node { + if let Some(query) = image + .image_search_query + .as_deref() + .map(str::trim) + .filter(|query| !query.is_empty()) + { + return query.to_string(); + } + } + + if is_image_placeholder_frame(node) { + if let Some(label) = placeholder_label_text(node) { + return label; + } + } + + if let Some(name) = node + .base() + .name + .as_deref() + .map(str::trim) + .filter(|name| !name.is_empty()) + { + if !is_generic_placeholder_name(name) { + return name.to_string(); + } + } + + if let Some(parent_name) = parent_semantic_name(parent_names) { + return parent_name; + } + + node.base() + .name + .as_deref() + .map(str::trim) + .filter(|name| !name.is_empty()) + .unwrap_or("placeholder") + .to_string() +} + +fn placeholder_label_text(node: &PenNode) -> Option { + let children = node.children()?; + for child in children { + let PenNode::Text(text) = child else { + continue; + }; + if text.base.role.as_deref() != Some("image-placeholder-label") { + continue; + } + let label = match &text.content { + TextContent::Plain(content) => content.trim().to_string(), + TextContent::Styled(segments) => segments + .iter() + .map(|segment| segment.text.as_str()) + .collect::() + .trim() + .to_string(), + }; + if !label.is_empty() { + return Some(label); + } + } + None +} + +fn parent_semantic_name(parent_names: &[String]) -> Option { + parent_names.iter().take(3).find_map(|name| { + let trimmed = name.trim(); + if trimmed.is_empty() + || is_generic_placeholder_name(trimmed) + || is_layout_context_name(trimmed) + { + return None; + } + Some(trimmed.to_string()) + }) +} + +fn is_generic_placeholder_name(name: &str) -> bool { + matches!( + name.trim().to_ascii_lowercase().as_str(), + "image" + | "photo" + | "cover" + | "hero" + | "thumbnail" + | "thumb" + | "picture" + | "banner" + | "poster" + | "image placeholder" + | "placeholder icon" + | "placeholder" + | "card image" + | "card photo" + | "product image" + | "item image" + ) +} + +fn is_layout_context_name(name: &str) -> bool { + name.split(|c: char| !c.is_ascii_alphanumeric()) + .map(str::to_ascii_lowercase) + .any(|word| { + matches!( + word.as_str(), + "card" + | "wrapper" + | "container" + | "section" + | "frame" + | "root" + | "page" + | "stack" + | "row" + | "column" + | "content" + ) + }) +} + pub(crate) fn apply_result(state: &mut EditorState, node_id: &NodeId, url: &str) -> bool { let url = url.trim(); if url.is_empty() { @@ -167,14 +394,36 @@ pub(crate) fn apply_result(state: &mut EditorState, node_id: &NodeId, url: &str) let Some(node) = walkers::find_node_mut(state.active_children_mut(), node_id) else { return false; }; - let PenNode::Image(image) = node else { - return false; - }; - if image.src == url { - return false; + let is_unfilled_placeholder_frame = is_frame_placeholder_still_unfilled(node); + match node { + PenNode::Image(image) => { + if image.src == url { + return false; + } + image.src = url.to_string(); + true + } + PenNode::Frame(frame) if is_unfilled_placeholder_frame => { + frame.container.fill = Some(vec![PenFill::Image(ImageFillBody { + url: url.to_string(), + mode: Some(ImageFillMode::Crop), + original_size: None, + transform: None, + explain: None, + opacity: None, + exposure: None, + contrast: None, + saturation: None, + temperature: None, + tint: None, + highlights: None, + shadows: None, + })]); + frame.children = Some(Vec::new()); + true + } + _ => false, } - image.src = url.to_string(); - true } fn fetch_first_image_url_blocking( @@ -315,99 +564,5 @@ async fn fetch_wikimedia(client: &reqwest::Client, query: &str) -> Option) -> PenNode { - PenNode::Image(ImageNode { - base: PenNodeBase { - id: id.to_string(), - name: Some("Menu photo".into()), - ..Default::default() - }, - src: src.to_string(), - object_fit: None, - width: Some(SizingBehavior::Number(240.0)), - height: Some(SizingBehavior::Number(160.0)), - corner_radius: None, - effects: None, - exposure: None, - contrast: None, - saturation: None, - temperature: None, - tint: None, - highlights: None, - shadows: None, - image_prompt: None, - image_search_query: query.map(str::to_string), - state: None, - bindings: None, - events: None, - lifecycle: None, - semantics: None, - gestures: None, - route: None, - }) - } - - #[test] - fn collect_targets_prefers_query_on_empty_image_nodes() { - let mut state = EditorState::default(); - state.active_children_mut().clear(); - state - .active_children_mut() - .push(image_node("img1", "", Some("burger fries"))); - - let targets = collect_targets(&state, &HashSet::new()); - - assert_eq!(targets.len(), 1); - assert_eq!(targets[0].node_id.as_str(), "img1"); - assert_eq!(targets[0].query, "burger fries"); - } - - #[test] - fn apply_result_sets_empty_image_src() { - let mut state = EditorState::default(); - state.active_children_mut().clear(); - state - .active_children_mut() - .push(image_node("img1", "", Some("burger fries"))); - - assert!(apply_result( - &mut state, - &NodeId::new("img1"), - "https://example.com/photo.jpg" - )); - let PenNode::Image(image) = &state.active_children()[0] else { - panic!("expected image"); - }; - assert_eq!(image.src, "https://example.com/photo.jpg"); - } - - #[test] - fn openverse_credentials_require_both_fields() { - let mut state = EditorState::default(); - assert!(OpenverseCredentials::from_state(&state).is_none()); - - state.editor_ui.agent_settings.openverse_client_id = " client ".into(); - assert!(OpenverseCredentials::from_state(&state).is_none()); - - state.editor_ui.agent_settings.openverse_client_secret = " secret ".into(); - let credentials = OpenverseCredentials::from_state(&state).expect("complete credentials"); - assert_eq!(credentials.client_id, "client"); - assert_eq!(credentials.client_secret, "secret"); - } - - #[tokio::test] - #[ignore = "network smoke test for Openverse/Wikimedia"] - async fn fetch_first_image_url_smoke() { - let url = fetch_first_image_url("burger fries", None) - .await - .expect("common query should return an image URL"); - assert!(url.starts_with("http"), "got {url}"); - } -} +#[path = "image_search_session_tests.rs"] +mod tests; diff --git a/crates/op-host-desktop/src/image_search_session_tests.rs b/crates/op-host-desktop/src/image_search_session_tests.rs new file mode 100644 index 000000000..69f66e990 --- /dev/null +++ b/crates/op-host-desktop/src/image_search_session_tests.rs @@ -0,0 +1,241 @@ +use super::*; +use jian_ops_schema::node::base::PenNodeBase; +use jian_ops_schema::node::{ContainerProps, FrameNode, ImageNode, PenNode, TextContent, TextNode}; +use jian_ops_schema::sizing::SizingBehavior; +use jian_ops_schema::style::{ImageFillMode, PenFill, SolidFillBody}; + +fn image_node(id: &str, src: &str, query: Option<&str>) -> PenNode { + PenNode::Image(ImageNode { + base: PenNodeBase { + id: id.to_string(), + name: Some("Menu photo".into()), + ..Default::default() + }, + src: src.to_string(), + object_fit: None, + width: Some(SizingBehavior::Number(240.0)), + height: Some(SizingBehavior::Number(160.0)), + corner_radius: None, + effects: None, + exposure: None, + contrast: None, + saturation: None, + temperature: None, + tint: None, + highlights: None, + shadows: None, + image_prompt: None, + image_search_query: query.map(str::to_string), + state: None, + bindings: None, + events: None, + lifecycle: None, + semantics: None, + gestures: None, + route: None, + }) +} + +fn text_label(id: &str, role: Option<&str>, content: &str) -> PenNode { + PenNode::Text(TextNode { + base: PenNodeBase { + id: id.to_string(), + name: Some("Label".into()), + role: role.map(str::to_string), + ..Default::default() + }, + width: Some(SizingBehavior::Number(160.0)), + height: Some(SizingBehavior::Number(24.0)), + content: TextContent::Plain(content.to_string()), + font_family: None, + font_size: None, + font_weight: None, + font_style: None, + letter_spacing: None, + line_height: None, + text_align: None, + text_align_vertical: None, + text_growth: None, + underline: None, + strikethrough: None, + fill: None, + effects: None, + state: None, + bindings: None, + events: None, + lifecycle: None, + semantics: None, + gestures: None, + route: None, + }) +} + +fn frame_node( + id: &str, + name: &str, + role: Option<&str>, + fill: Option>, + children: Vec, +) -> PenNode { + PenNode::Frame(FrameNode { + base: PenNodeBase { + id: id.to_string(), + name: Some(name.into()), + role: role.map(str::to_string), + ..Default::default() + }, + container: ContainerProps { + width: Some(SizingBehavior::Number(240.0)), + height: Some(SizingBehavior::Number(160.0)), + fill, + ..Default::default() + }, + children: Some(children), + reusable: None, + slot: None, + state: None, + bindings: None, + events: None, + lifecycle: None, + semantics: None, + gestures: None, + route: None, + }) +} + +fn solid_fill() -> PenFill { + PenFill::Solid(SolidFillBody { + color: "#E5E7EB".into(), + explain: None, + opacity: None, + blend_mode: None, + }) +} + +#[test] +fn collect_targets_prefers_query_on_empty_image_nodes() { + let mut state = EditorState::default(); + state.active_children_mut().clear(); + state + .active_children_mut() + .push(image_node("img1", "", Some("burger fries"))); + + let targets = collect_targets(&state, &HashSet::new()); + + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].node_id.as_str(), "img1"); + assert_eq!(targets[0].query, "burger fries"); +} + +#[test] +fn apply_result_sets_empty_image_src() { + let mut state = EditorState::default(); + state.active_children_mut().clear(); + state + .active_children_mut() + .push(image_node("img1", "", Some("burger fries"))); + + assert!(apply_result( + &mut state, + &NodeId::new("img1"), + "https://example.com/photo.jpg" + )); + let PenNode::Image(image) = &state.active_children()[0] else { + panic!("expected image"); + }; + assert_eq!(image.src, "https://example.com/photo.jpg"); +} + +#[test] +fn collect_targets_includes_unfilled_placeholder_frames() { + let mut state = EditorState::default(); + state.active_children_mut().clear(); + state.active_children_mut().push(frame_node( + "photo", + "Image", + Some("image-placeholder"), + Some(vec![solid_fill()]), + vec![text_label( + "label", + Some("image-placeholder-label"), + "pizza hero", + )], + )); + + let targets = collect_targets(&state, &HashSet::new()); + + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].node_id.as_str(), "photo"); + assert_eq!(targets[0].query, "pizza hero"); +} + +#[test] +fn collect_targets_uses_parent_semantic_name_for_generic_heuristic_frame() { + let mut state = EditorState::default(); + state.active_children_mut().clear(); + let photo = frame_node("photo", "Image", None, Some(vec![solid_fill()]), Vec::new()); + state + .active_children_mut() + .push(frame_node("card", "Bella Italia", None, None, vec![photo])); + + let targets = collect_targets(&state, &HashSet::new()); + + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].node_id.as_str(), "photo"); + assert_eq!(targets[0].query, "Bella Italia"); +} + +#[test] +fn apply_result_repaints_placeholder_frame_with_image_fill_and_clears_children() { + let mut state = EditorState::default(); + state.active_children_mut().clear(); + state.active_children_mut().push(frame_node( + "photo", + "Image", + Some("image-placeholder"), + Some(vec![solid_fill()]), + vec![text_label( + "label", + Some("image-placeholder-label"), + "pizza hero", + )], + )); + + assert!(apply_result( + &mut state, + &NodeId::new("photo"), + "https://example.com/photo.jpg" + )); + let PenNode::Frame(frame) = &state.active_children()[0] else { + panic!("expected frame"); + }; + let Some([PenFill::Image(image_fill)]) = frame.container.fill.as_deref() else { + panic!("expected single image fill"); + }; + assert_eq!(image_fill.url, "https://example.com/photo.jpg"); + assert_eq!(image_fill.mode, Some(ImageFillMode::Crop)); + assert_eq!(frame.children.as_deref(), Some(&[][..])); +} + +#[test] +fn openverse_credentials_require_both_fields() { + let mut state = EditorState::default(); + assert!(OpenverseCredentials::from_state(&state).is_none()); + + state.editor_ui.agent_settings.openverse_client_id = " client ".into(); + assert!(OpenverseCredentials::from_state(&state).is_none()); + + state.editor_ui.agent_settings.openverse_client_secret = " secret ".into(); + let credentials = OpenverseCredentials::from_state(&state).expect("complete credentials"); + assert_eq!(credentials.client_id, "client"); + assert_eq!(credentials.client_secret, "secret"); +} + +#[tokio::test] +#[ignore = "network smoke test for Openverse/Wikimedia"] +async fn fetch_first_image_url_smoke() { + let url = fetch_first_image_url("burger fries", None) + .await + .expect("common query should return an image URL"); + assert!(url.starts_with("http"), "got {url}"); +}