From 854e1642d87c4a316d5489567081e05acf642818 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Wed, 29 Jul 2026 14:31:38 +0300 Subject: [PATCH 1/3] feat(fig): prototype lazy population worker - Populate subsequent lazy pages in a development-only persistent worker - Journal created, updated, and deleted graph state as field deltas - Invalidate stale worker replicas after authoritative graph mutations - Fall back to synchronous population when the worker is unavailable --- packages/core/src/editor/pages.ts | 43 +++++- packages/core/src/io/formats/fig/read.ts | 11 +- packages/core/src/kiwi/fig/parse/worker.ts | 68 +++++++-- .../core/src/kiwi/fig/population/client.ts | 140 ++++++++++++++++++ .../core/src/kiwi/fig/population/delta.ts | 133 +++++++++++++++++ packages/fig/src/instance-overrides/index.ts | 1 - src/app/editor/session/modules.ts | 5 +- tests/e2e/pages/fig-population-worker.spec.ts | 65 ++++++++ .../io/fig/export/lazy-population.test.ts | 40 +++++ .../io/fig/import/population-delta.test.ts | 63 ++++++++ 10 files changed, 552 insertions(+), 17 deletions(-) create mode 100644 packages/core/src/kiwi/fig/population/client.ts create mode 100644 packages/core/src/kiwi/fig/population/delta.ts create mode 100644 tests/e2e/pages/fig-population-worker.spec.ts create mode 100644 tests/engine/io/fig/export/lazy-population.test.ts create mode 100644 tests/engine/io/fig/import/population-delta.test.ts diff --git a/packages/core/src/editor/pages.ts b/packages/core/src/editor/pages.ts index 18600b599..bd2d8ec74 100644 --- a/packages/core/src/editor/pages.ts +++ b/packages/core/src/editor/pages.ts @@ -1,6 +1,10 @@ import type { Color } from '@open-pencil/scene-graph/primitives' import { populateLazyFigImportRoots } from '#core/kiwi/fig/lazy-import' +import { + canUseFigPopulationWorker, + createFigPopulationWorker +} from '#core/kiwi/fig/population/client' import { computeAllLayouts } from '#core/layout' import { fontManager } from '#core/text/fonts' import { collectGraphFontRequirements } from '#core/text/requirements' @@ -11,10 +15,20 @@ import type { EditorContext } from './types' export function createPageActions(ctx: EditorContext) { const pageViewportStore = createPageViewportStore(ctx) + let populationWorkerInstance: ReturnType | undefined + let populationWorkerGeneration = 0 + let pageSwitchGeneration = 0 + + function populationWorker() { + if (!canUseFigPopulationWorker(ctx.graph)) return null + populationWorkerInstance ??= createFigPopulationWorker(ctx.graph) + return populationWorkerInstance + } async function switchPage(pageId: string) { const page = ctx.graph.getNode(pageId) if (page?.type !== 'CANVAS') return + const switchGeneration = ++pageSwitchGeneration pageViewportStore.saveCurrentPageViewport() @@ -26,7 +40,24 @@ export function createPageActions(ctx: EditorContext) { pageViewportStore.restorePageViewport(pageId) - const populated = populateLazyFigImportRoots(ctx.graph, [pageId]) + ctx.state.loading = true + let populated: boolean + try { + const worker = populationWorker() + const workerGeneration = populationWorkerGeneration + const workerResult = worker ? await worker.populate(pageId) : null + if (workerGeneration !== populationWorkerGeneration) return + if (workerResult === null) { + worker?.terminate() + populationWorkerInstance = undefined + populated = populateLazyFigImportRoots(ctx.graph, [pageId]) + } else { + populated = workerResult + } + } finally { + if (switchGeneration === pageSwitchGeneration) ctx.state.loading = false + } + if (switchGeneration !== pageSwitchGeneration) return const childIds = ctx.graph.getChildren(pageId).map((node) => node.id) const toLoad = fontManager.collectFontKeys(ctx.graph, childIds) @@ -58,6 +89,14 @@ export function createPageActions(ctx: EditorContext) { ctx.requestRender() } + function clearPageViewports() { + populationWorkerGeneration++ + pageSwitchGeneration++ + populationWorkerInstance?.terminate() + populationWorkerInstance = undefined + pageViewportStore.clearPageViewports() + } + function addPage(name?: string) { const pages = ctx.graph.getPages() const pageName = name ?? `Page ${pages.length + 1}` @@ -106,6 +145,6 @@ export function createPageActions(ctx: EditorContext) { movePage, renamePage, setPageColor, - clearPageViewports: pageViewportStore.clearPageViewports + clearPageViewports } } diff --git a/packages/core/src/io/formats/fig/read.ts b/packages/core/src/io/formats/fig/read.ts index 8d2817bfa..932403ac3 100644 --- a/packages/core/src/io/formats/fig/read.ts +++ b/packages/core/src/io/formats/fig/read.ts @@ -5,6 +5,7 @@ import { IS_BROWSER } from '#core/constants' import { importNodeChanges } from '#core/kiwi/fig/import' import { deserializeSceneGraph } from '#core/kiwi/fig/parse/transfer' import type { SerializedSceneGraph } from '#core/kiwi/fig/parse/transfer' +import { registerFigPopulationWorker } from '#core/kiwi/fig/population/client' export interface ParseFigFileOptions { populate?: 'all' | 'first-page' | 'none' @@ -36,12 +37,18 @@ function parseViaWorker(buffer: ArrayBuffer, options: ParseFigFileOptions): Prom }) worker.onmessage = (e: MessageEvent) => { - worker.terminate() if (e.data.error || !e.data.graph) { + worker.terminate() reject(new Error(e.data.error ?? 'Worker failed to parse .fig file')) return } - resolve(deserializeSceneGraph(e.data.graph)) + const graph = deserializeSceneGraph(e.data.graph) + if (options.populate === 'first-page') { + worker.onmessage = null + worker.onerror = null + registerFigPopulationWorker(graph, worker) + } else worker.terminate() + resolve(graph) } worker.onerror = (err) => { diff --git a/packages/core/src/kiwi/fig/parse/worker.ts b/packages/core/src/kiwi/fig/parse/worker.ts index 96691d427..e03d4864b 100644 --- a/packages/core/src/kiwi/fig/parse/worker.ts +++ b/packages/core/src/kiwi/fig/parse/worker.ts @@ -1,35 +1,81 @@ import { parseFigBuffer } from '@open-pencil/fig' +import type { SceneGraph } from '@open-pencil/scene-graph' import { importNodeChanges } from '#core/kiwi/fig/import' +import { getLazyFigImportContext, populateLazyFigImportRoots } from '#core/kiwi/fig/lazy-import' import { serializeSceneGraph, serializedSceneGraphTransferList } from '#core/kiwi/fig/parse/transfer' +import { buildFigPopulationDelta, installFigMutationJournal } from '#core/kiwi/fig/population/delta' interface WorkerParseRequest { buffer: ArrayBuffer options?: { populate?: 'all' | 'first-page' } } +interface PopulateRequest { + type: 'populate' + requestId: string + baseRevision: number + pageId: string +} +type WorkerRequest = ArrayBuffer | WorkerParseRequest | PopulateRequest +type WorkerPostMessage = (message: unknown, transfer: Transferable[]) => void +const postWorkerMessage: WorkerPostMessage = (message, transfer) => { + globalThis.postMessage(message, { transfer }) +} +let graph: SceneGraph | undefined -type WorkerScope = typeof self & { - postMessage(message: unknown, transfer: Transferable[]): void +function isPopulateRequest(request: WorkerRequest): request is PopulateRequest { + return !(request instanceof ArrayBuffer) && 'type' in request } -self.onmessage = (e: MessageEvent) => { +self.onmessage = (event: MessageEvent) => { try { - const request = e.data instanceof ArrayBuffer ? { buffer: e.data } : e.data + const request = event.data + if (isPopulateRequest(request)) { + if (!graph) throw new Error('FIG parse worker has no retained graph') + const journal = installFigMutationJournal(graph) + try { + const populated = populateLazyFigImportRoots(graph, [request.pageId]) + const context = getLazyFigImportContext(graph) + if (!context) throw new Error('FIG population worker has no lazy import context') + postWorkerMessage( + { + type: 'population-result', + requestId: request.requestId, + baseRevision: request.baseRevision, + populated, + delta: buildFigPopulationDelta(graph, journal, context.populatedRootIds) + }, + [] + ) + } finally { + journal.stop() + } + return + } + const parseRequest: WorkerParseRequest = + request instanceof ArrayBuffer ? { buffer: request } : request const { nodeChanges, blobs, images, figKiwiVersion, figSchemaDeflated } = parseFigBuffer( - request.buffer + parseRequest.buffer ) - const graph = importNodeChanges(nodeChanges, blobs, new Map(images), request.options) + graph = importNodeChanges(nodeChanges, blobs, new Map(images), parseRequest.options) graph.figKiwiVersion = figKiwiVersion graph.figSchemaDeflated = figSchemaDeflated const serialized = serializeSceneGraph(graph) - ;(self as WorkerScope).postMessage( - { graph: serialized }, - serializedSceneGraphTransferList(serialized) + const transfer = + parseRequest.options?.populate === 'first-page' + ? [] + : serializedSceneGraphTransferList(serialized) + postWorkerMessage({ graph: serialized }, transfer) + } catch (error) { + postWorkerMessage( + { + type: 'population-error', + error: error instanceof Error ? error.message : String(error) + }, + [] ) - } catch (err) { - self.postMessage({ error: err instanceof Error ? err.message : String(err) }) } } diff --git a/packages/core/src/kiwi/fig/population/client.ts b/packages/core/src/kiwi/fig/population/client.ts new file mode 100644 index 000000000..3a057f1bb --- /dev/null +++ b/packages/core/src/kiwi/fig/population/client.ts @@ -0,0 +1,140 @@ +import type { SceneGraph } from '@open-pencil/scene-graph' + +import { getLazyFigImportContext } from '#core/kiwi/fig/lazy-import' +import { randomHex } from '#core/random' + +import { applyFigPopulationDelta, type FigPopulationDelta } from './delta' + +interface PopulationResult { + type: 'population-result' + requestId: string + baseRevision: number + populated: boolean + delta: FigPopulationDelta +} +type WorkerResult = PopulationResult | { type: 'population-error'; error: string } + +const MAX_FIG_POPULATION_WORKER_NODES = 200_000 +const populationWorkers = new WeakMap() + +export interface FigPopulationWorkerTelemetry { + event: 'registered' | 'populate' | 'fallback' | 'stale' | 'terminated' + reason?: 'oversized' | 'graph-mutation' | 'worker-error' + durationMs?: number + applyMs?: number + created?: number + updated?: number + deleted?: number +} + +function emitTelemetry(detail: FigPopulationWorkerTelemetry): void { + if (typeof globalThis.dispatchEvent !== 'function') return + globalThis.dispatchEvent(new CustomEvent('openpencil:fig-population-worker', { detail })) +} + +export function registerFigPopulationWorker(graph: SceneGraph, worker: Worker): void { + if (graph.nodes.size > MAX_FIG_POPULATION_WORKER_NODES) { + emitTelemetry({ event: 'fallback', reason: 'oversized' }) + worker.terminate() + return + } + populationWorkers.set(graph, worker) + emitTelemetry({ event: 'registered' }) +} + +export function canUseFigPopulationWorker(graph: SceneGraph): boolean { + return ( + import.meta.env.DEV && + populationWorkers.has(graph) && + getLazyFigImportContext(graph) !== undefined + ) +} + +export interface FigPopulationWorker { + populate: (pageId: string) => Promise + terminate: () => void +} + +export function createFigPopulationWorker(graph: SceneGraph): FigPopulationWorker | null { + if (!canUseFigPopulationWorker(graph)) return null + const worker = populationWorkers.get(graph) + if (!worker) return null + const pending = new Map< + string, + { resolve: (value: boolean | null) => void; revision: number; startedAt: number } + >() + let revision = 0 + let stale = false + let applyingDelta = false + const invalidate = () => { + if (applyingDelta || stale) return + revision++ + stale = true + emitTelemetry({ event: 'stale', reason: 'graph-mutation' }) + } + const unbind = graph.onNodeEvents({ + created: invalidate, + updated: invalidate, + deleted: invalidate, + reparented: invalidate, + reordered: invalidate + }) + const fail = (emit = true) => { + stale = true + if (emit) emitTelemetry({ event: 'fallback', reason: 'worker-error' }) + for (const request of pending.values()) request.resolve(null) + pending.clear() + worker.terminate() + populationWorkers.delete(graph) + } + worker.onmessage = (event: MessageEvent) => { + const result = event.data + if (result.type === 'population-error') return fail() + const request = pending.get(result.requestId) + if (!request) return + pending.delete(result.requestId) + if (stale || revision !== request.revision || result.baseRevision !== request.revision) { + emitTelemetry({ event: 'stale', reason: 'graph-mutation' }) + return request.resolve(null) + } + applyingDelta = true + const applyStartedAt = performance.now() + try { + applyFigPopulationDelta(graph, result.delta) + const context = getLazyFigImportContext(graph) + if (context) context.populatedRootIds = new Set(result.delta.populatedRootIds) + } catch { + applyingDelta = false + fail() + return request.resolve(null) + } finally { + applyingDelta = false + } + request.resolve(result.populated) + emitTelemetry({ + event: 'populate', + durationMs: performance.now() - request.startedAt, + applyMs: performance.now() - applyStartedAt, + created: result.delta.created.length, + updated: result.delta.updated.length, + deleted: result.delta.deleted.length + }) + } + worker.onerror = () => fail() + return { + populate(pageId) { + if (stale) return Promise.resolve(null) + const requestId = randomHex() + const baseRevision = revision + return new Promise((resolve) => { + pending.set(requestId, { resolve, revision: baseRevision, startedAt: performance.now() }) + worker.postMessage({ type: 'populate', requestId, baseRevision, pageId }, []) + }) + }, + terminate() { + emitTelemetry({ event: 'terminated' }) + unbind() + fail(false) + } + } +} diff --git a/packages/core/src/kiwi/fig/population/delta.ts b/packages/core/src/kiwi/fig/population/delta.ts new file mode 100644 index 000000000..9d39bb7a4 --- /dev/null +++ b/packages/core/src/kiwi/fig/population/delta.ts @@ -0,0 +1,133 @@ +import { isEqual } from 'es-toolkit/predicate' + +import type { SceneGraph, SceneNode } from '@open-pencil/scene-graph' + +export interface FigPopulationDelta { + created: Array<[string, SceneNode]> + updated: Array<[string, Partial]> + deleted: string[] + instanceIndex: Array<[string, string[]]> + populatedRootIds: string[] +} + +export interface FigMutationJournal { + before: Map> + created: Set + deleted: Set + stop: () => void +} + +export function installFigMutationJournal(graph: SceneGraph): FigMutationJournal { + const existingAtStart = new Set(graph.nodes.keys()) + const before = new Map>() + const created = new Set() + const deleted = new Set() + const original = { + createNode: graph.createNode.bind(graph), + createNodeWithId: graph.createNodeWithId.bind(graph), + updateNode: graph.updateNode.bind(graph), + deleteNode: graph.deleteNode.bind(graph) + } + function touch(id: string | null | undefined, fields: Iterable): void { + if (!id || created.has(id)) return + const node = graph.getNode(id) + if (!node) return + const snapshot = before.get(id) ?? {} + before.set(id, snapshot) + for (const field of fields) { + if (!(field in snapshot)) Object.assign(snapshot, { [field]: structuredClone(node[field]) }) + } + } + graph.createNode = ((type, parentId, overrides) => { + touch(parentId, ['childIds']) + const node = original.createNode(type, parentId, overrides) + created.add(node.id) + return node + }) as SceneGraph['createNode'] + graph.createNodeWithId = ((id, type, parentId, overrides) => { + touch(parentId, ['childIds']) + const node = original.createNodeWithId(id, type, parentId, overrides) + created.add(node.id) + return node + }) as SceneGraph['createNodeWithId'] + graph.updateNode = ((id, changes) => { + const node = graph.getNode(id) + if (node) { + const fields = (Object.keys(changes) as (keyof SceneNode)[]).filter( + (field) => !isEqual(node[field], changes[field]) + ) + if (fields.length > 0) fields.push('source') + if ('componentId' in changes) fields.push('componentId') + if ('fills' in changes || 'strokes' in changes) fields.push('boundVariables') + touch(id, fields) + } + original.updateNode(id, changes) + }) as SceneGraph['updateNode'] + graph.deleteNode = ((id) => { + const node = graph.getNode(id) + touch(node?.parentId, ['childIds']) + const pending = node ? [id] : [] + while (pending.length > 0) { + const currentId = pending.pop() + if (!currentId) continue + const current = graph.getNode(currentId) + if (current) pending.push(...current.childIds) + if (!existingAtStart.has(currentId)) { + created.delete(currentId) + before.delete(currentId) + } else deleted.add(currentId) + } + original.deleteNode(id) + }) as SceneGraph['deleteNode'] + return { + before, + created, + deleted, + stop() { + graph.createNode = original.createNode + graph.createNodeWithId = original.createNodeWithId + graph.updateNode = original.updateNode + graph.deleteNode = original.deleteNode + } + } +} + +export function buildFigPopulationDelta( + graph: SceneGraph, + journal: FigMutationJournal, + populatedRootIds: Iterable +): FigPopulationDelta { + const updated: Array<[string, Partial]> = [] + for (const [id, previous] of journal.before) { + const current = graph.getNode(id) + if (!current || journal.deleted.has(id)) continue + const changes: Partial = {} + for (const key of Object.keys(previous) as (keyof SceneNode)[]) { + if (!isEqual(previous[key], current[key])) + Object.assign(changes, { [key]: structuredClone(current[key]) }) + } + if (Object.keys(changes).length > 0) updated.push([id, changes]) + } + const created = [...journal.created] + .map((id) => graph.getNode(id)) + .filter((node) => node !== undefined) + .map((node) => [node.id, structuredClone(node)] as [string, SceneNode]) + return { + created, + updated, + deleted: [...journal.deleted], + instanceIndex: [...graph.instanceIndex].map(([id, ids]) => [id, [...ids]]), + populatedRootIds: [...populatedRootIds] + } +} + +export function applyFigPopulationDelta(graph: SceneGraph, delta: FigPopulationDelta): void { + graph.preserveSourceMetadataDuring(() => { + for (const [, node] of delta.created) { + graph.createNodeWithId(node.id, node.type, node.parentId, node) + } + for (const [id, changes] of delta.updated) graph.updateNode(id, changes) + for (const id of delta.deleted) graph.deleteNode(id) + }) + graph.instanceIndex = new Map(delta.instanceIndex.map(([id, ids]) => [id, new Set(ids)])) +} diff --git a/packages/fig/src/instance-overrides/index.ts b/packages/fig/src/instance-overrides/index.ts index a719bcca3..e377954c3 100644 --- a/packages/fig/src/instance-overrides/index.ts +++ b/packages/fig/src/instance-overrides/index.ts @@ -39,7 +39,6 @@ import { applySymbolOverrides } from './symbol/overrides' import { propagateNodePropsTransitively, propagateOverridesTransitively } from './sync' import { indexCloneNodes } from './sync/sources' import type { InstanceNodeChange, OverrideContext, ComponentPropValue } from './types' -import { overrideCandidates } from './utils' /** * Identify nodes whose kiwi NC has explicit property values that DIFFER diff --git a/src/app/editor/session/modules.ts b/src/app/editor/session/modules.ts index e1bde7300..7e24e85df 100644 --- a/src/app/editor/session/modules.ts +++ b/src/app/editor/session/modules.ts @@ -80,7 +80,10 @@ export function createEditorStoreModules( setStorageDocumentSource: documentIO.setStorageDocumentSource, setPlannedFilePath: documentIO.setPlannedFilePath, startWatchingCurrentFile: documentIO.startWatchingCurrentFile, - dispose: documentIO.disposeDocumentIO, + dispose: () => { + editor.clearPageViewports() + documentIO.disposeDocumentIO() + }, ...documentExport, ...mobileClipboard, ...profiler diff --git a/tests/e2e/pages/fig-population-worker.spec.ts b/tests/e2e/pages/fig-population-worker.spec.ts new file mode 100644 index 000000000..f980bfef4 --- /dev/null +++ b/tests/e2e/pages/fig-population-worker.spec.ts @@ -0,0 +1,65 @@ +import { expect, test, useEditorSetup } from '#tests/e2e/fixtures' + +const editor = useEditorSetup() + +test('populates a real lazy FIG page in the retained parse worker', async () => { + await editor.page.evaluate(() => { + const events: unknown[] = [] + Object.assign(window, { figPopulationWorkerEvents: events }) + window.addEventListener('openpencil:fig-population-worker', (event) => { + if (event instanceof CustomEvent) events.push(event.detail) + }) + }) + const openFile = editor.page.evaluate(() => + window.openPencil?.openFile?.('/tests/fixtures/gold-preview.fig') + ) + await openFile + await editor.canvas.waitForRender() + + const targetPageId = await editor.page.evaluate(() => { + const store = window.openPencil?.getStore?.() + if (!store) throw new Error('OpenPencil store not initialized') + return store.graph.getPages(true).at(-1)?.id + }) + if (!targetPageId) throw new Error('Target page not found') + + const loading = editor.page.getByTestId('canvas-loading') + const switchPromise = editor.page.evaluate((pageId) => { + const store = window.openPencil?.getStore?.() + if (!store) throw new Error('OpenPencil store not initialized') + return store.switchPage(pageId) + }, targetPageId) + await expect(loading).toBeVisible() + await switchPromise + await expect(loading).not.toBeVisible() + + const currentPage = await editor.page.evaluate(() => { + const store = window.openPencil?.getStore?.() + if (!store) throw new Error('OpenPencil store not initialized') + return { + currentPageId: store.state.currentPageId, + childCount: store.graph.getChildren(store.state.currentPageId).length + } + }) + expect(currentPage.currentPageId).toBe(targetPageId) + expect(currentPage.childCount).toBeGreaterThan(0) + + const events = await editor.page.evaluate( + () => Reflect.get(window, 'figPopulationWorkerEvents') as Array<{ event: string }> + ) + expect(events.map(({ event }) => event)).toEqual( + expect.arrayContaining(['registered', 'populate']) + ) + + await editor.page.evaluate(() => window.openPencil?.getStore?.()?.dispose()) + await expect + .poll(() => + editor.page.evaluate( + () => + (Reflect.get(window, 'figPopulationWorkerEvents') as Array<{ event: string }>).at(-1) + ?.event + ) + ) + .toBe('terminated') + editor.canvas.assertNoErrors() +}) diff --git a/tests/engine/io/fig/export/lazy-population.test.ts b/tests/engine/io/fig/export/lazy-population.test.ts new file mode 100644 index 000000000..785ab334c --- /dev/null +++ b/tests/engine/io/fig/export/lazy-population.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from 'bun:test' + +import { exportFigFile, initCodec, parseFigFile } from '@open-pencil/core' +import { setLazyFigImportContext } from '@open-pencil/core/kiwi/fig/lazy-import' +import { SceneGraph } from '@open-pencil/scene-graph' + +function lazyExportGraph() { + const graph = new SceneGraph() + const firstPage = graph.getPages()[0] + const secondPage = graph.addPage('Second') + const component = graph.createNode('COMPONENT', firstPage.id, { name: 'Button' }) + graph.createNode('TEXT', component.id, { text: 'Label' }) + const instance = graph.createNode('INSTANCE', secondPage.id, { + name: 'Button instance', + componentId: component.id + }) + setLazyFigImportContext(graph, { + changeMap: new Map(), + guidToNodeId: new Map(), + blobs: [], + populatedRootIds: new Set([firstPage.id]) + }) + return { graph, secondPage, instance } +} + +describe('FIG population export lifecycle', () => { + test('exports all remaining lazy pages after a partial visit', async () => { + await initCodec() + const { graph, instance } = lazyExportGraph() + expect(graph.getChildren(instance.id)).toHaveLength(0) + + const exported = await exportFigFile(graph) + const reimported = await parseFigFile(exported.buffer as ArrayBuffer, { populate: 'all' }) + const reimportedInstance = [...reimported.getAllNodes()].find( + (node) => node.type === 'INSTANCE' && node.name === 'Button instance' + ) + expect(reimportedInstance).toBeDefined() + expect(reimported.getChildren(reimportedInstance?.id ?? '')).toHaveLength(1) + }) +}) diff --git a/tests/engine/io/fig/import/population-delta.test.ts b/tests/engine/io/fig/import/population-delta.test.ts new file mode 100644 index 000000000..30b6b80fc --- /dev/null +++ b/tests/engine/io/fig/import/population-delta.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from 'bun:test' + +import { SceneGraph } from '@open-pencil/scene-graph' + +import { + applyFigPopulationDelta, + buildFigPopulationDelta, + installFigMutationJournal +} from '#core/kiwi/fig/population/delta' + +describe('FIG population deltas', () => { + test('captures created, updated, and deleted nodes', () => { + const graph = new SceneGraph() + const page = graph.getPages()[0] + const updated = graph.createNode('RECTANGLE', page.id, { name: 'Before' }) + const deleted = graph.createNode('RECTANGLE', page.id, { name: 'Deleted' }) + const journal = installFigMutationJournal(graph) + graph.updateNode(updated.id, { name: 'After', x: 12 }) + graph.deleteNode(deleted.id) + const created = graph.createNode('TEXT', page.id, { text: 'Created' }) + journal.stop() + const delta = buildFigPopulationDelta(graph, journal, [page.id]) + expect(delta.created.map(([id]) => id)).toEqual([created.id]) + expect(delta.updated).toContainEqual([ + updated.id, + expect.objectContaining({ name: 'After', x: 12 }) + ]) + expect(delta.deleted).toEqual([deleted.id]) + }) + + test('applies created, updated, deleted, indexed, and event-visible changes', () => { + const source = new SceneGraph() + const page = source.getPages()[0] + const updated = source.createNode('RECTANGLE', page.id, { name: 'Before' }) + const deleted = source.createNode('RECTANGLE', page.id, { name: 'Deleted' }) + const target = new SceneGraph() + target.rootId = source.rootId + target.nodes = structuredClone(source.nodes) + const journal = installFigMutationJournal(source) + source.updateNode(updated.id, { name: 'After', visible: false }) + source.deleteNode(deleted.id) + const component = source.createNode('COMPONENT', page.id, { name: 'Component' }) + const created = source.createNode('INSTANCE', page.id, { componentId: component.id }) + journal.stop() + const delta = buildFigPopulationDelta(source, journal, [page.id]) + const events: string[] = [] + target.onNodeEvents({ + created: (node) => events.push(`created:${node.id}`), + updated: (id) => events.push(`updated:${id}`), + deleted: (id) => events.push(`deleted:${id}`) + }) + + applyFigPopulationDelta(target, delta) + + expect(target.getNode(updated.id)).toMatchObject({ name: 'After', visible: false }) + expect(target.getNode(deleted.id)).toBeUndefined() + expect(target.getNode(created.id)).toMatchObject({ componentId: component.id }) + expect(target.instanceIndex.get(component.id)).toEqual(new Set([created.id])) + expect(events).toContain(`updated:${updated.id}`) + expect(events).toContain(`deleted:${deleted.id}`) + expect(events).toContain(`created:${created.id}`) + }) +}) From a5a744843fb256d4a98f08227477ab8916535403 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Fri, 7 Aug 2026 09:36:46 +0300 Subject: [PATCH 2/3] fix(fig): harden population worker lifecycle - Register invalidation immediately and release subscriptions on every failure path\n- Bound worker requests and settle parse and population errors consistently\n- Clear stale page loading state and retain current instance override imports --- packages/core/src/editor/pages.ts | 1 + packages/core/src/io/formats/fig/read.ts | 19 +++--- packages/core/src/kiwi/fig/parse/worker.ts | 10 +-- .../core/src/kiwi/fig/population/client.ts | 62 ++++++++++++++----- packages/fig/src/instance-overrides/index.ts | 1 + 5 files changed, 64 insertions(+), 29 deletions(-) diff --git a/packages/core/src/editor/pages.ts b/packages/core/src/editor/pages.ts index bd2d8ec74..0085e2a5b 100644 --- a/packages/core/src/editor/pages.ts +++ b/packages/core/src/editor/pages.ts @@ -92,6 +92,7 @@ export function createPageActions(ctx: EditorContext) { function clearPageViewports() { populationWorkerGeneration++ pageSwitchGeneration++ + ctx.state.loading = false populationWorkerInstance?.terminate() populationWorkerInstance = undefined pageViewportStore.clearPageViewports() diff --git a/packages/core/src/io/formats/fig/read.ts b/packages/core/src/io/formats/fig/read.ts index 932403ac3..81aff70a0 100644 --- a/packages/core/src/io/formats/fig/read.ts +++ b/packages/core/src/io/formats/fig/read.ts @@ -42,13 +42,18 @@ function parseViaWorker(buffer: ArrayBuffer, options: ParseFigFileOptions): Prom reject(new Error(e.data.error ?? 'Worker failed to parse .fig file')) return } - const graph = deserializeSceneGraph(e.data.graph) - if (options.populate === 'first-page') { - worker.onmessage = null - worker.onerror = null - registerFigPopulationWorker(graph, worker) - } else worker.terminate() - resolve(graph) + try { + const graph = deserializeSceneGraph(e.data.graph) + if (options.populate === 'first-page') { + worker.onmessage = null + worker.onerror = null + registerFigPopulationWorker(graph, worker) + } else worker.terminate() + resolve(graph) + } catch (error) { + worker.terminate() + reject(error instanceof Error ? error : new Error(String(error))) + } } worker.onerror = (err) => { diff --git a/packages/core/src/kiwi/fig/parse/worker.ts b/packages/core/src/kiwi/fig/parse/worker.ts index e03d4864b..42dac3324 100644 --- a/packages/core/src/kiwi/fig/parse/worker.ts +++ b/packages/core/src/kiwi/fig/parse/worker.ts @@ -31,8 +31,8 @@ function isPopulateRequest(request: WorkerRequest): request is PopulateRequest { } self.onmessage = (event: MessageEvent) => { + const request = event.data try { - const request = event.data if (isPopulateRequest(request)) { if (!graph) throw new Error('FIG parse worker has no retained graph') const journal = installFigMutationJournal(graph) @@ -70,11 +70,11 @@ self.onmessage = (event: MessageEvent) => { : serializedSceneGraphTransferList(serialized) postWorkerMessage({ graph: serialized }, transfer) } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) postWorkerMessage( - { - type: 'population-error', - error: error instanceof Error ? error.message : String(error) - }, + isPopulateRequest(request) + ? { type: 'population-error', error: errorMessage } + : { error: errorMessage }, [] ) } diff --git a/packages/core/src/kiwi/fig/population/client.ts b/packages/core/src/kiwi/fig/population/client.ts index 3a057f1bb..302522354 100644 --- a/packages/core/src/kiwi/fig/population/client.ts +++ b/packages/core/src/kiwi/fig/population/client.ts @@ -15,7 +15,8 @@ interface PopulationResult { type WorkerResult = PopulationResult | { type: 'population-error'; error: string } const MAX_FIG_POPULATION_WORKER_NODES = 200_000 -const populationWorkers = new WeakMap() +const FIG_POPULATION_WORKER_TIMEOUT_MS = 30_000 +const populationWorkers = new WeakMap() export interface FigPopulationWorkerTelemetry { event: 'registered' | 'populate' | 'fallback' | 'stale' | 'terminated' @@ -38,13 +39,18 @@ export function registerFigPopulationWorker(graph: SceneGraph, worker: Worker): worker.terminate() return } - populationWorkers.set(graph, worker) + const client = createPopulationWorkerClient(graph, worker) + populationWorkers.set(graph, client) emitTelemetry({ event: 'registered' }) } +function isDevelopmentBuild(meta: { env?: { DEV?: boolean } }): boolean { + return meta.env?.DEV ?? false +} + export function canUseFigPopulationWorker(graph: SceneGraph): boolean { return ( - import.meta.env.DEV && + isDevelopmentBuild(import.meta) && populationWorkers.has(graph) && getLazyFigImportContext(graph) !== undefined ) @@ -57,11 +63,18 @@ export interface FigPopulationWorker { export function createFigPopulationWorker(graph: SceneGraph): FigPopulationWorker | null { if (!canUseFigPopulationWorker(graph)) return null - const worker = populationWorkers.get(graph) - if (!worker) return null + return populationWorkers.get(graph) ?? null +} + +function createPopulationWorkerClient(graph: SceneGraph, worker: Worker): FigPopulationWorker { const pending = new Map< string, - { resolve: (value: boolean | null) => void; revision: number; startedAt: number } + { + resolve: (value: boolean | null) => void + revision: number + startedAt: number + timeout: ReturnType + } >() let revision = 0 let stale = false @@ -72,26 +85,36 @@ export function createFigPopulationWorker(graph: SceneGraph): FigPopulationWorke stale = true emitTelemetry({ event: 'stale', reason: 'graph-mutation' }) } - const unbind = graph.onNodeEvents({ + let unbind: (() => void) | undefined + const releaseSubscription = () => { + unbind?.() + unbind = undefined + } + const fail = (emit = true) => { + stale = true + if (emit) emitTelemetry({ event: 'fallback', reason: 'worker-error' }) + for (const request of pending.values()) { + clearTimeout(request.timeout) + request.resolve(null) + } + pending.clear() + releaseSubscription() + worker.terminate() + populationWorkers.delete(graph) + } + unbind = graph.onNodeEvents({ created: invalidate, updated: invalidate, deleted: invalidate, reparented: invalidate, reordered: invalidate }) - const fail = (emit = true) => { - stale = true - if (emit) emitTelemetry({ event: 'fallback', reason: 'worker-error' }) - for (const request of pending.values()) request.resolve(null) - pending.clear() - worker.terminate() - populationWorkers.delete(graph) - } worker.onmessage = (event: MessageEvent) => { const result = event.data if (result.type === 'population-error') return fail() const request = pending.get(result.requestId) if (!request) return + clearTimeout(request.timeout) pending.delete(result.requestId) if (stale || revision !== request.revision || result.baseRevision !== request.revision) { emitTelemetry({ event: 'stale', reason: 'graph-mutation' }) @@ -127,13 +150,18 @@ export function createFigPopulationWorker(graph: SceneGraph): FigPopulationWorke const requestId = randomHex() const baseRevision = revision return new Promise((resolve) => { - pending.set(requestId, { resolve, revision: baseRevision, startedAt: performance.now() }) + const timeout = setTimeout(() => fail(), FIG_POPULATION_WORKER_TIMEOUT_MS) + pending.set(requestId, { + resolve, + revision: baseRevision, + startedAt: performance.now(), + timeout + }) worker.postMessage({ type: 'populate', requestId, baseRevision, pageId }, []) }) }, terminate() { emitTelemetry({ event: 'terminated' }) - unbind() fail(false) } } diff --git a/packages/fig/src/instance-overrides/index.ts b/packages/fig/src/instance-overrides/index.ts index e377954c3..a719bcca3 100644 --- a/packages/fig/src/instance-overrides/index.ts +++ b/packages/fig/src/instance-overrides/index.ts @@ -39,6 +39,7 @@ import { applySymbolOverrides } from './symbol/overrides' import { propagateNodePropsTransitively, propagateOverridesTransitively } from './sync' import { indexCloneNodes } from './sync/sources' import type { InstanceNodeChange, OverrideContext, ComponentPropValue } from './types' +import { overrideCandidates } from './utils' /** * Identify nodes whose kiwi NC has explicit property values that DIFFER From 02400a84b9d581fa9a0143c0ea7ab236872a2f60 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Fri, 7 Aug 2026 14:43:59 +0300 Subject: [PATCH 3/3] fix(fig): finalize worker cleanup semantics - Retain worker graphs only for lazy first-page imports\n- Make termination idempotent\n- Wait on deterministic population telemetry in E2E coverage --- packages/core/src/kiwi/fig/parse/worker.ts | 9 +++++---- packages/core/src/kiwi/fig/population/client.ts | 3 +++ tests/e2e/pages/fig-population-worker.spec.ts | 15 ++++++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/core/src/kiwi/fig/parse/worker.ts b/packages/core/src/kiwi/fig/parse/worker.ts index 42dac3324..9826b3018 100644 --- a/packages/core/src/kiwi/fig/parse/worker.ts +++ b/packages/core/src/kiwi/fig/parse/worker.ts @@ -60,10 +60,11 @@ self.onmessage = (event: MessageEvent) => { const { nodeChanges, blobs, images, figKiwiVersion, figSchemaDeflated } = parseFigBuffer( parseRequest.buffer ) - graph = importNodeChanges(nodeChanges, blobs, new Map(images), parseRequest.options) - graph.figKiwiVersion = figKiwiVersion - graph.figSchemaDeflated = figSchemaDeflated - const serialized = serializeSceneGraph(graph) + const parsedGraph = importNodeChanges(nodeChanges, blobs, new Map(images), parseRequest.options) + parsedGraph.figKiwiVersion = figKiwiVersion + parsedGraph.figSchemaDeflated = figSchemaDeflated + graph = parseRequest.options?.populate === 'first-page' ? parsedGraph : undefined + const serialized = serializeSceneGraph(parsedGraph) const transfer = parseRequest.options?.populate === 'first-page' ? [] diff --git a/packages/core/src/kiwi/fig/population/client.ts b/packages/core/src/kiwi/fig/population/client.ts index 302522354..f806502b9 100644 --- a/packages/core/src/kiwi/fig/population/client.ts +++ b/packages/core/src/kiwi/fig/population/client.ts @@ -78,6 +78,7 @@ function createPopulationWorkerClient(graph: SceneGraph, worker: Worker): FigPop >() let revision = 0 let stale = false + let disposed = false let applyingDelta = false const invalidate = () => { if (applyingDelta || stale) return @@ -161,6 +162,8 @@ function createPopulationWorkerClient(graph: SceneGraph, worker: Worker): FigPop }) }, terminate() { + if (disposed) return + disposed = true emitTelemetry({ event: 'terminated' }) fail(false) } diff --git a/tests/e2e/pages/fig-population-worker.spec.ts b/tests/e2e/pages/fig-population-worker.spec.ts index f980bfef4..a585a1005 100644 --- a/tests/e2e/pages/fig-population-worker.spec.ts +++ b/tests/e2e/pages/fig-population-worker.spec.ts @@ -23,15 +23,20 @@ test('populates a real lazy FIG page in the retained parse worker', async () => }) if (!targetPageId) throw new Error('Target page not found') - const loading = editor.page.getByTestId('canvas-loading') - const switchPromise = editor.page.evaluate((pageId) => { + await editor.page.evaluate((pageId) => { const store = window.openPencil?.getStore?.() if (!store) throw new Error('OpenPencil store not initialized') return store.switchPage(pageId) }, targetPageId) - await expect(loading).toBeVisible() - await switchPromise - await expect(loading).not.toBeVisible() + await expect + .poll(() => + editor.page.evaluate(() => + (Reflect.get(window, 'figPopulationWorkerEvents') as Array<{ event: string }>).some( + ({ event }) => event === 'populate' + ) + ) + ) + .toBe(true) const currentPage = await editor.page.evaluate(() => { const store = window.openPencil?.getStore?.()