Merge pull request #435 from open-pencil/fig-population-worker

feat(fig): prototype lazy population worker
This commit is contained in:
Danila Poyarkov 2026-08-07 15:02:38 +03:00 committed by GitHub
commit f972efe229
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 598 additions and 19 deletions

View file

@ -1,6 +1,10 @@
import type { Color } from '@open-pencil/scene-graph/primitives'
import { populateLazyFigImportRoots } from '#core/kiwi/fig/lazy-import'
import {
canUseFigPopulationWorker,
createFigPopulationWorker
} from '#core/kiwi/fig/population/client'
import { computeAllLayouts } from '#core/layout'
import { fontManager } from '#core/text/fonts'
import { collectGraphFontRequirements } from '#core/text/requirements'
@ -11,10 +15,20 @@ import type { EditorContext } from './types'
export function createPageActions(ctx: EditorContext) {
const pageViewportStore = createPageViewportStore(ctx)
let populationWorkerInstance: ReturnType<typeof createFigPopulationWorker> | undefined
let populationWorkerGeneration = 0
let pageSwitchGeneration = 0
function populationWorker() {
if (!canUseFigPopulationWorker(ctx.graph)) return null
populationWorkerInstance ??= createFigPopulationWorker(ctx.graph)
return populationWorkerInstance
}
async function switchPage(pageId: string) {
const page = ctx.graph.getNode(pageId)
if (page?.type !== 'CANVAS') return
const switchGeneration = ++pageSwitchGeneration
pageViewportStore.saveCurrentPageViewport()
@ -26,7 +40,24 @@ export function createPageActions(ctx: EditorContext) {
pageViewportStore.restorePageViewport(pageId)
const populated = populateLazyFigImportRoots(ctx.graph, [pageId])
ctx.state.loading = true
let populated: boolean
try {
const worker = populationWorker()
const workerGeneration = populationWorkerGeneration
const workerResult = worker ? await worker.populate(pageId) : null
if (workerGeneration !== populationWorkerGeneration) return
if (workerResult === null) {
worker?.terminate()
populationWorkerInstance = undefined
populated = populateLazyFigImportRoots(ctx.graph, [pageId])
} else {
populated = workerResult
}
} finally {
if (switchGeneration === pageSwitchGeneration) ctx.state.loading = false
}
if (switchGeneration !== pageSwitchGeneration) return
const childIds = ctx.graph.getChildren(pageId).map((node) => node.id)
const toLoad = fontManager.collectFontKeys(ctx.graph, childIds)
@ -58,6 +89,15 @@ export function createPageActions(ctx: EditorContext) {
ctx.requestRender()
}
function clearPageViewports() {
populationWorkerGeneration++
pageSwitchGeneration++
ctx.state.loading = false
populationWorkerInstance?.terminate()
populationWorkerInstance = undefined
pageViewportStore.clearPageViewports()
}
function addPage(name?: string) {
const pages = ctx.graph.getPages()
const pageName = name ?? `Page ${pages.length + 1}`
@ -106,6 +146,6 @@ export function createPageActions(ctx: EditorContext) {
movePage,
renamePage,
setPageColor,
clearPageViewports: pageViewportStore.clearPageViewports
clearPageViewports
}
}

View file

@ -5,6 +5,7 @@ import { IS_BROWSER } from '#core/constants'
import { importNodeChanges } from '#core/kiwi/fig/import'
import { deserializeSceneGraph } from '#core/kiwi/fig/parse/transfer'
import type { SerializedSceneGraph } from '#core/kiwi/fig/parse/transfer'
import { registerFigPopulationWorker } from '#core/kiwi/fig/population/client'
export interface ParseFigFileOptions {
populate?: 'all' | 'first-page' | 'none'
@ -36,12 +37,23 @@ function parseViaWorker(buffer: ArrayBuffer, options: ParseFigFileOptions): Prom
})
worker.onmessage = (e: MessageEvent<WorkerParseResult>) => {
worker.terminate()
if (e.data.error || !e.data.graph) {
worker.terminate()
reject(new Error(e.data.error ?? 'Worker failed to parse .fig file'))
return
}
resolve(deserializeSceneGraph(e.data.graph))
try {
const graph = deserializeSceneGraph(e.data.graph)
if (options.populate === 'first-page') {
worker.onmessage = null
worker.onerror = null
registerFigPopulationWorker(graph, worker)
} else worker.terminate()
resolve(graph)
} catch (error) {
worker.terminate()
reject(error instanceof Error ? error : new Error(String(error)))
}
}
worker.onerror = (err) => {

View file

@ -1,35 +1,82 @@
import { parseFigBuffer } from '@open-pencil/fig'
import type { SceneGraph } from '@open-pencil/scene-graph'
import { importNodeChanges } from '#core/kiwi/fig/import'
import { getLazyFigImportContext, populateLazyFigImportRoots } from '#core/kiwi/fig/lazy-import'
import {
serializeSceneGraph,
serializedSceneGraphTransferList
} from '#core/kiwi/fig/parse/transfer'
import { buildFigPopulationDelta, installFigMutationJournal } from '#core/kiwi/fig/population/delta'
interface WorkerParseRequest {
buffer: ArrayBuffer
options?: { populate?: 'all' | 'first-page' }
}
interface PopulateRequest {
type: 'populate'
requestId: string
baseRevision: number
pageId: string
}
type WorkerRequest = ArrayBuffer | WorkerParseRequest | PopulateRequest
type WorkerPostMessage = (message: unknown, transfer: Transferable[]) => void
const postWorkerMessage: WorkerPostMessage = (message, transfer) => {
globalThis.postMessage(message, { transfer })
}
let graph: SceneGraph | undefined
type WorkerScope = typeof self & {
postMessage(message: unknown, transfer: Transferable[]): void
function isPopulateRequest(request: WorkerRequest): request is PopulateRequest {
return !(request instanceof ArrayBuffer) && 'type' in request
}
self.onmessage = (e: MessageEvent<ArrayBuffer | WorkerParseRequest>) => {
self.onmessage = (event: MessageEvent<WorkerRequest>) => {
const request = event.data
try {
const request = e.data instanceof ArrayBuffer ? { buffer: e.data } : e.data
if (isPopulateRequest(request)) {
if (!graph) throw new Error('FIG parse worker has no retained graph')
const journal = installFigMutationJournal(graph)
try {
const populated = populateLazyFigImportRoots(graph, [request.pageId])
const context = getLazyFigImportContext(graph)
if (!context) throw new Error('FIG population worker has no lazy import context')
postWorkerMessage(
{
type: 'population-result',
requestId: request.requestId,
baseRevision: request.baseRevision,
populated,
delta: buildFigPopulationDelta(graph, journal, context.populatedRootIds)
},
[]
)
} finally {
journal.stop()
}
return
}
const parseRequest: WorkerParseRequest =
request instanceof ArrayBuffer ? { buffer: request } : request
const { nodeChanges, blobs, images, figKiwiVersion, figSchemaDeflated } = parseFigBuffer(
request.buffer
parseRequest.buffer
)
const graph = importNodeChanges(nodeChanges, blobs, new Map(images), request.options)
graph.figKiwiVersion = figKiwiVersion
graph.figSchemaDeflated = figSchemaDeflated
const serialized = serializeSceneGraph(graph)
;(self as WorkerScope).postMessage(
{ graph: serialized },
serializedSceneGraphTransferList(serialized)
const parsedGraph = importNodeChanges(nodeChanges, blobs, new Map(images), parseRequest.options)
parsedGraph.figKiwiVersion = figKiwiVersion
parsedGraph.figSchemaDeflated = figSchemaDeflated
graph = parseRequest.options?.populate === 'first-page' ? parsedGraph : undefined
const serialized = serializeSceneGraph(parsedGraph)
const transfer =
parseRequest.options?.populate === 'first-page'
? []
: serializedSceneGraphTransferList(serialized)
postWorkerMessage({ graph: serialized }, transfer)
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error)
postWorkerMessage(
isPopulateRequest(request)
? { type: 'population-error', error: errorMessage }
: { error: errorMessage },
[]
)
} catch (err) {
self.postMessage({ error: err instanceof Error ? err.message : String(err) })
}
}

View file

@ -0,0 +1,171 @@
import type { SceneGraph } from '@open-pencil/scene-graph'
import { getLazyFigImportContext } from '#core/kiwi/fig/lazy-import'
import { randomHex } from '#core/random'
import { applyFigPopulationDelta, type FigPopulationDelta } from './delta'
interface PopulationResult {
type: 'population-result'
requestId: string
baseRevision: number
populated: boolean
delta: FigPopulationDelta
}
type WorkerResult = PopulationResult | { type: 'population-error'; error: string }
const MAX_FIG_POPULATION_WORKER_NODES = 200_000
const FIG_POPULATION_WORKER_TIMEOUT_MS = 30_000
const populationWorkers = new WeakMap<SceneGraph, FigPopulationWorker>()
export interface FigPopulationWorkerTelemetry {
event: 'registered' | 'populate' | 'fallback' | 'stale' | 'terminated'
reason?: 'oversized' | 'graph-mutation' | 'worker-error'
durationMs?: number
applyMs?: number
created?: number
updated?: number
deleted?: number
}
function emitTelemetry(detail: FigPopulationWorkerTelemetry): void {
if (typeof globalThis.dispatchEvent !== 'function') return
globalThis.dispatchEvent(new CustomEvent('openpencil:fig-population-worker', { detail }))
}
export function registerFigPopulationWorker(graph: SceneGraph, worker: Worker): void {
if (graph.nodes.size > MAX_FIG_POPULATION_WORKER_NODES) {
emitTelemetry({ event: 'fallback', reason: 'oversized' })
worker.terminate()
return
}
const client = createPopulationWorkerClient(graph, worker)
populationWorkers.set(graph, client)
emitTelemetry({ event: 'registered' })
}
function isDevelopmentBuild(meta: { env?: { DEV?: boolean } }): boolean {
return meta.env?.DEV ?? false
}
export function canUseFigPopulationWorker(graph: SceneGraph): boolean {
return (
isDevelopmentBuild(import.meta) &&
populationWorkers.has(graph) &&
getLazyFigImportContext(graph) !== undefined
)
}
export interface FigPopulationWorker {
populate: (pageId: string) => Promise<boolean | null>
terminate: () => void
}
export function createFigPopulationWorker(graph: SceneGraph): FigPopulationWorker | null {
if (!canUseFigPopulationWorker(graph)) return null
return populationWorkers.get(graph) ?? null
}
function createPopulationWorkerClient(graph: SceneGraph, worker: Worker): FigPopulationWorker {
const pending = new Map<
string,
{
resolve: (value: boolean | null) => void
revision: number
startedAt: number
timeout: ReturnType<typeof setTimeout>
}
>()
let revision = 0
let stale = false
let disposed = false
let applyingDelta = false
const invalidate = () => {
if (applyingDelta || stale) return
revision++
stale = true
emitTelemetry({ event: 'stale', reason: 'graph-mutation' })
}
let unbind: (() => void) | undefined
const releaseSubscription = () => {
unbind?.()
unbind = undefined
}
const fail = (emit = true) => {
stale = true
if (emit) emitTelemetry({ event: 'fallback', reason: 'worker-error' })
for (const request of pending.values()) {
clearTimeout(request.timeout)
request.resolve(null)
}
pending.clear()
releaseSubscription()
worker.terminate()
populationWorkers.delete(graph)
}
unbind = graph.onNodeEvents({
created: invalidate,
updated: invalidate,
deleted: invalidate,
reparented: invalidate,
reordered: invalidate
})
worker.onmessage = (event: MessageEvent<WorkerResult>) => {
const result = event.data
if (result.type === 'population-error') return fail()
const request = pending.get(result.requestId)
if (!request) return
clearTimeout(request.timeout)
pending.delete(result.requestId)
if (stale || revision !== request.revision || result.baseRevision !== request.revision) {
emitTelemetry({ event: 'stale', reason: 'graph-mutation' })
return request.resolve(null)
}
applyingDelta = true
const applyStartedAt = performance.now()
try {
applyFigPopulationDelta(graph, result.delta)
const context = getLazyFigImportContext(graph)
if (context) context.populatedRootIds = new Set(result.delta.populatedRootIds)
} catch {
applyingDelta = false
fail()
return request.resolve(null)
} finally {
applyingDelta = false
}
request.resolve(result.populated)
emitTelemetry({
event: 'populate',
durationMs: performance.now() - request.startedAt,
applyMs: performance.now() - applyStartedAt,
created: result.delta.created.length,
updated: result.delta.updated.length,
deleted: result.delta.deleted.length
})
}
worker.onerror = () => fail()
return {
populate(pageId) {
if (stale) return Promise.resolve(null)
const requestId = randomHex()
const baseRevision = revision
return new Promise((resolve) => {
const timeout = setTimeout(() => fail(), FIG_POPULATION_WORKER_TIMEOUT_MS)
pending.set(requestId, {
resolve,
revision: baseRevision,
startedAt: performance.now(),
timeout
})
worker.postMessage({ type: 'populate', requestId, baseRevision, pageId }, [])
})
},
terminate() {
if (disposed) return
disposed = true
emitTelemetry({ event: 'terminated' })
fail(false)
}
}
}

View file

@ -0,0 +1,133 @@
import { isEqual } from 'es-toolkit/predicate'
import type { SceneGraph, SceneNode } from '@open-pencil/scene-graph'
export interface FigPopulationDelta {
created: Array<[string, SceneNode]>
updated: Array<[string, Partial<SceneNode>]>
deleted: string[]
instanceIndex: Array<[string, string[]]>
populatedRootIds: string[]
}
export interface FigMutationJournal {
before: Map<string, Partial<SceneNode>>
created: Set<string>
deleted: Set<string>
stop: () => void
}
export function installFigMutationJournal(graph: SceneGraph): FigMutationJournal {
const existingAtStart = new Set(graph.nodes.keys())
const before = new Map<string, Partial<SceneNode>>()
const created = new Set<string>()
const deleted = new Set<string>()
const original = {
createNode: graph.createNode.bind(graph),
createNodeWithId: graph.createNodeWithId.bind(graph),
updateNode: graph.updateNode.bind(graph),
deleteNode: graph.deleteNode.bind(graph)
}
function touch(id: string | null | undefined, fields: Iterable<keyof SceneNode>): void {
if (!id || created.has(id)) return
const node = graph.getNode(id)
if (!node) return
const snapshot = before.get(id) ?? {}
before.set(id, snapshot)
for (const field of fields) {
if (!(field in snapshot)) Object.assign(snapshot, { [field]: structuredClone(node[field]) })
}
}
graph.createNode = ((type, parentId, overrides) => {
touch(parentId, ['childIds'])
const node = original.createNode(type, parentId, overrides)
created.add(node.id)
return node
}) as SceneGraph['createNode']
graph.createNodeWithId = ((id, type, parentId, overrides) => {
touch(parentId, ['childIds'])
const node = original.createNodeWithId(id, type, parentId, overrides)
created.add(node.id)
return node
}) as SceneGraph['createNodeWithId']
graph.updateNode = ((id, changes) => {
const node = graph.getNode(id)
if (node) {
const fields = (Object.keys(changes) as (keyof SceneNode)[]).filter(
(field) => !isEqual(node[field], changes[field])
)
if (fields.length > 0) fields.push('source')
if ('componentId' in changes) fields.push('componentId')
if ('fills' in changes || 'strokes' in changes) fields.push('boundVariables')
touch(id, fields)
}
original.updateNode(id, changes)
}) as SceneGraph['updateNode']
graph.deleteNode = ((id) => {
const node = graph.getNode(id)
touch(node?.parentId, ['childIds'])
const pending = node ? [id] : []
while (pending.length > 0) {
const currentId = pending.pop()
if (!currentId) continue
const current = graph.getNode(currentId)
if (current) pending.push(...current.childIds)
if (!existingAtStart.has(currentId)) {
created.delete(currentId)
before.delete(currentId)
} else deleted.add(currentId)
}
original.deleteNode(id)
}) as SceneGraph['deleteNode']
return {
before,
created,
deleted,
stop() {
graph.createNode = original.createNode
graph.createNodeWithId = original.createNodeWithId
graph.updateNode = original.updateNode
graph.deleteNode = original.deleteNode
}
}
}
export function buildFigPopulationDelta(
graph: SceneGraph,
journal: FigMutationJournal,
populatedRootIds: Iterable<string>
): FigPopulationDelta {
const updated: Array<[string, Partial<SceneNode>]> = []
for (const [id, previous] of journal.before) {
const current = graph.getNode(id)
if (!current || journal.deleted.has(id)) continue
const changes: Partial<SceneNode> = {}
for (const key of Object.keys(previous) as (keyof SceneNode)[]) {
if (!isEqual(previous[key], current[key]))
Object.assign(changes, { [key]: structuredClone(current[key]) })
}
if (Object.keys(changes).length > 0) updated.push([id, changes])
}
const created = [...journal.created]
.map((id) => graph.getNode(id))
.filter((node) => node !== undefined)
.map((node) => [node.id, structuredClone(node)] as [string, SceneNode])
return {
created,
updated,
deleted: [...journal.deleted],
instanceIndex: [...graph.instanceIndex].map(([id, ids]) => [id, [...ids]]),
populatedRootIds: [...populatedRootIds]
}
}
export function applyFigPopulationDelta(graph: SceneGraph, delta: FigPopulationDelta): void {
graph.preserveSourceMetadataDuring(() => {
for (const [, node] of delta.created) {
graph.createNodeWithId(node.id, node.type, node.parentId, node)
}
for (const [id, changes] of delta.updated) graph.updateNode(id, changes)
for (const id of delta.deleted) graph.deleteNode(id)
})
graph.instanceIndex = new Map(delta.instanceIndex.map(([id, ids]) => [id, new Set(ids)]))
}

View file

@ -80,7 +80,10 @@ export function createEditorStoreModules(
setStorageDocumentSource: documentIO.setStorageDocumentSource,
setPlannedFilePath: documentIO.setPlannedFilePath,
startWatchingCurrentFile: documentIO.startWatchingCurrentFile,
dispose: documentIO.disposeDocumentIO,
dispose: () => {
editor.clearPageViewports()
documentIO.disposeDocumentIO()
},
...documentExport,
...mobileClipboard,
...profiler

View file

@ -0,0 +1,70 @@
import { expect, test, useEditorSetup } from '#tests/e2e/fixtures'
const editor = useEditorSetup()
test('populates a real lazy FIG page in the retained parse worker', async () => {
await editor.page.evaluate(() => {
const events: unknown[] = []
Object.assign(window, { figPopulationWorkerEvents: events })
window.addEventListener('openpencil:fig-population-worker', (event) => {
if (event instanceof CustomEvent) events.push(event.detail)
})
})
const openFile = editor.page.evaluate(() =>
window.openPencil?.openFile?.('/tests/fixtures/gold-preview.fig')
)
await openFile
await editor.canvas.waitForRender()
const targetPageId = await editor.page.evaluate(() => {
const store = window.openPencil?.getStore?.()
if (!store) throw new Error('OpenPencil store not initialized')
return store.graph.getPages(true).at(-1)?.id
})
if (!targetPageId) throw new Error('Target page not found')
await editor.page.evaluate((pageId) => {
const store = window.openPencil?.getStore?.()
if (!store) throw new Error('OpenPencil store not initialized')
return store.switchPage(pageId)
}, targetPageId)
await expect
.poll(() =>
editor.page.evaluate(() =>
(Reflect.get(window, 'figPopulationWorkerEvents') as Array<{ event: string }>).some(
({ event }) => event === 'populate'
)
)
)
.toBe(true)
const currentPage = await editor.page.evaluate(() => {
const store = window.openPencil?.getStore?.()
if (!store) throw new Error('OpenPencil store not initialized')
return {
currentPageId: store.state.currentPageId,
childCount: store.graph.getChildren(store.state.currentPageId).length
}
})
expect(currentPage.currentPageId).toBe(targetPageId)
expect(currentPage.childCount).toBeGreaterThan(0)
const events = await editor.page.evaluate(
() => Reflect.get(window, 'figPopulationWorkerEvents') as Array<{ event: string }>
)
expect(events.map(({ event }) => event)).toEqual(
expect.arrayContaining(['registered', 'populate'])
)
await editor.page.evaluate(() => window.openPencil?.getStore?.()?.dispose())
await expect
.poll(() =>
editor.page.evaluate(
() =>
(Reflect.get(window, 'figPopulationWorkerEvents') as Array<{ event: string }>).at(-1)
?.event
)
)
.toBe('terminated')
editor.canvas.assertNoErrors()
})

View file

@ -0,0 +1,40 @@
import { describe, expect, test } from 'bun:test'
import { exportFigFile, initCodec, parseFigFile } from '@open-pencil/core'
import { setLazyFigImportContext } from '@open-pencil/core/kiwi/fig/lazy-import'
import { SceneGraph } from '@open-pencil/scene-graph'
function lazyExportGraph() {
const graph = new SceneGraph()
const firstPage = graph.getPages()[0]
const secondPage = graph.addPage('Second')
const component = graph.createNode('COMPONENT', firstPage.id, { name: 'Button' })
graph.createNode('TEXT', component.id, { text: 'Label' })
const instance = graph.createNode('INSTANCE', secondPage.id, {
name: 'Button instance',
componentId: component.id
})
setLazyFigImportContext(graph, {
changeMap: new Map(),
guidToNodeId: new Map(),
blobs: [],
populatedRootIds: new Set([firstPage.id])
})
return { graph, secondPage, instance }
}
describe('FIG population export lifecycle', () => {
test('exports all remaining lazy pages after a partial visit', async () => {
await initCodec()
const { graph, instance } = lazyExportGraph()
expect(graph.getChildren(instance.id)).toHaveLength(0)
const exported = await exportFigFile(graph)
const reimported = await parseFigFile(exported.buffer as ArrayBuffer, { populate: 'all' })
const reimportedInstance = [...reimported.getAllNodes()].find(
(node) => node.type === 'INSTANCE' && node.name === 'Button instance'
)
expect(reimportedInstance).toBeDefined()
expect(reimported.getChildren(reimportedInstance?.id ?? '')).toHaveLength(1)
})
})

View file

@ -0,0 +1,63 @@
import { describe, expect, test } from 'bun:test'
import { SceneGraph } from '@open-pencil/scene-graph'
import {
applyFigPopulationDelta,
buildFigPopulationDelta,
installFigMutationJournal
} from '#core/kiwi/fig/population/delta'
describe('FIG population deltas', () => {
test('captures created, updated, and deleted nodes', () => {
const graph = new SceneGraph()
const page = graph.getPages()[0]
const updated = graph.createNode('RECTANGLE', page.id, { name: 'Before' })
const deleted = graph.createNode('RECTANGLE', page.id, { name: 'Deleted' })
const journal = installFigMutationJournal(graph)
graph.updateNode(updated.id, { name: 'After', x: 12 })
graph.deleteNode(deleted.id)
const created = graph.createNode('TEXT', page.id, { text: 'Created' })
journal.stop()
const delta = buildFigPopulationDelta(graph, journal, [page.id])
expect(delta.created.map(([id]) => id)).toEqual([created.id])
expect(delta.updated).toContainEqual([
updated.id,
expect.objectContaining({ name: 'After', x: 12 })
])
expect(delta.deleted).toEqual([deleted.id])
})
test('applies created, updated, deleted, indexed, and event-visible changes', () => {
const source = new SceneGraph()
const page = source.getPages()[0]
const updated = source.createNode('RECTANGLE', page.id, { name: 'Before' })
const deleted = source.createNode('RECTANGLE', page.id, { name: 'Deleted' })
const target = new SceneGraph()
target.rootId = source.rootId
target.nodes = structuredClone(source.nodes)
const journal = installFigMutationJournal(source)
source.updateNode(updated.id, { name: 'After', visible: false })
source.deleteNode(deleted.id)
const component = source.createNode('COMPONENT', page.id, { name: 'Component' })
const created = source.createNode('INSTANCE', page.id, { componentId: component.id })
journal.stop()
const delta = buildFigPopulationDelta(source, journal, [page.id])
const events: string[] = []
target.onNodeEvents({
created: (node) => events.push(`created:${node.id}`),
updated: (id) => events.push(`updated:${id}`),
deleted: (id) => events.push(`deleted:${id}`)
})
applyFigPopulationDelta(target, delta)
expect(target.getNode(updated.id)).toMatchObject({ name: 'After', visible: false })
expect(target.getNode(deleted.id)).toBeUndefined()
expect(target.getNode(created.id)).toMatchObject({ componentId: component.id })
expect(target.instanceIndex.get(component.id)).toEqual(new Set([created.id]))
expect(events).toContain(`updated:${updated.id}`)
expect(events).toContain(`deleted:${deleted.id}`)
expect(events).toContain(`created:${created.id}`)
})
})