From d315547f2b7636e5483886fbb10743957e418833 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Sun, 26 Jul 2026 14:26:44 +0300 Subject: [PATCH] feat(app): register S3-compatible storage - Adapt the contributed S3 client behind the neutral storage provider contract - Resolve access credentials lazily for each storage operation - Preserve namespace isolation, CORS setup, pagination, progress, and metadata fallbacks Co-authored-by: Rob Coenen <753704+rcoenen@users.noreply.github.com> --- bun.lock | 3 + package.json | 1 + src/app/integrations/storage/index.ts | 4 + src/app/integrations/storage/namespace.ts | 28 ++ src/app/integrations/storage/providers.ts | 20 ++ src/app/integrations/storage/s3/adapter.ts | 254 ++++++++++++++ src/app/integrations/storage/s3/client.ts | 324 ++++++++++++++++++ src/app/integrations/storage/s3/cors.ts | 198 +++++++++++ src/app/integrations/storage/s3/fetch.ts | 49 +++ src/app/integrations/storage/s3/region.ts | 41 +++ src/app/integrations/storage/s3/types.ts | 15 + src/app/integrations/storage/types.ts | 2 + .../app/integrations/storage/cors.test.ts | 57 +++ .../integrations/storage/list-objects.test.ts | 44 +++ .../integrations/storage/namespace.test.ts | 33 ++ .../app/integrations/storage/region.test.ts | 23 ++ .../registry.test.ts} | 15 + 17 files changed, 1111 insertions(+) create mode 100644 src/app/integrations/storage/namespace.ts create mode 100644 src/app/integrations/storage/providers.ts create mode 100644 src/app/integrations/storage/s3/adapter.ts create mode 100644 src/app/integrations/storage/s3/client.ts create mode 100644 src/app/integrations/storage/s3/cors.ts create mode 100644 src/app/integrations/storage/s3/fetch.ts create mode 100644 src/app/integrations/storage/s3/region.ts create mode 100644 src/app/integrations/storage/s3/types.ts create mode 100644 tests/engine/app/integrations/storage/cors.test.ts create mode 100644 tests/engine/app/integrations/storage/list-objects.test.ts create mode 100644 tests/engine/app/integrations/storage/namespace.test.ts create mode 100644 tests/engine/app/integrations/storage/region.test.ts rename tests/engine/app/integrations/{storage-registry.test.ts => storage/registry.test.ts} (89%) diff --git a/bun.lock b/bun.lock index fc58f487b..edac95972 100644 --- a/bun.lock +++ b/bun.lock @@ -36,6 +36,7 @@ "@unhead/vue": "^2.1.10", "@vueuse/core": "^14.2.1", "ai": "^6.0.174", + "aws4fetch": "^1.0.20", "canvaskit-wasm": "^0.40.0", "culori": "^4.0.2", "dedent": "^1.7.1", @@ -1498,6 +1499,8 @@ "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], + "aws4fetch": ["aws4fetch@1.0.20", "", {}, "sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g=="], + "axe-core": ["axe-core@4.12.1", "", {}, "sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA=="], "babel-plugin-polyfill-corejs2": ["babel-plugin-polyfill-corejs2@0.4.17", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-define-polyfill-provider": "^0.6.8", "semver": "^6.3.1" }, "peerDependencies": { "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" } }, "sha512-aTyf30K/rqAsNwN76zYrdtx8obu0E4KoUME29B1xj+B3WxgvWkp943vYQ+z8Mv3lw9xHXMHpvSPOBxzAkIa94w=="], diff --git a/package.json b/package.json index 2256b6a4a..df912d973 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "@unhead/vue": "^2.1.10", "@vueuse/core": "^14.2.1", "ai": "^6.0.174", + "aws4fetch": "^1.0.20", "canvaskit-wasm": "^0.40.0", "culori": "^4.0.2", "dedent": "^1.7.1", diff --git a/src/app/integrations/storage/index.ts b/src/app/integrations/storage/index.ts index 4fcff3550..0e67057bd 100644 --- a/src/app/integrations/storage/index.ts +++ b/src/app/integrations/storage/index.ts @@ -1,4 +1,8 @@ +export { S3_STORAGE_PROVIDER, storageProviderRegistry } from './providers' export { defineStorageProvider, StorageProviderRegistry } from './registry' +export { createS3StorageAdapter } from './s3/adapter' +export type { S3StorageAdapter } from './s3/adapter' +export type { S3CompatibleConfig, S3ConnectionResult } from './s3/types' export type { StorageAdapter, StorageAdapterContext, diff --git a/src/app/integrations/storage/namespace.ts b/src/app/integrations/storage/namespace.ts new file mode 100644 index 000000000..0c3e750a5 --- /dev/null +++ b/src/app/integrations/storage/namespace.ts @@ -0,0 +1,28 @@ +/** Fixed OpenPencil namespace inside a shared storage backend. */ +export const STORAGE_NAMESPACE = 'open_pencil_storage' +export const STORAGE_NAMESPACE_MARKER = `${STORAGE_NAMESPACE}/.openpencil-namespace` +export const STORAGE_DOCUMENTS_PREFIX = `${STORAGE_NAMESPACE}/canvases/` + +export function documentFigKey(documentId: string): string { + return `${STORAGE_DOCUMENTS_PREFIX}${documentId}.fig` +} + +export function documentMetaKey(documentId: string): string { + return `${STORAGE_DOCUMENTS_PREFIX}${documentId}.meta.json` +} + +export function documentThumbnailKey(documentId: string): string { + return `${STORAGE_DOCUMENTS_PREFIX}${documentId}.thumb.jpg` +} + +export function documentIdFromFigKey(key: string): string | null { + if (!key.startsWith(STORAGE_DOCUMENTS_PREFIX) || !key.endsWith('.fig')) return null + const id = key.slice(STORAGE_DOCUMENTS_PREFIX.length, -'.fig'.length) + if (!id || id.includes('/')) return null + return id +} + +export const NAMESPACE_MARKER_BODY = JSON.stringify({ + app: 'open-pencil', + version: 1 +}) diff --git a/src/app/integrations/storage/providers.ts b/src/app/integrations/storage/providers.ts new file mode 100644 index 000000000..75bee95e9 --- /dev/null +++ b/src/app/integrations/storage/providers.ts @@ -0,0 +1,20 @@ +import { defineStorageProvider, StorageProviderRegistry } from './registry' +import { createS3StorageAdapter } from './s3/adapter' + +export const S3_STORAGE_PROVIDER = defineStorageProvider({ + id: 's3-compatible', + label: 'S3 compatible', + description: 'AWS S3, Backblaze B2, Cloudflare R2, MinIO, and compatible storage', + preferenceFields: [ + { id: 'endpoint', label: 'Endpoint', kind: 'url', required: true }, + { id: 'bucket', label: 'Bucket', kind: 'text', required: true }, + { id: 'region', label: 'Region', kind: 'text' } + ], + credentialFields: [ + { id: 'access-key-id', label: 'Access key ID', required: true }, + { id: 'secret-access-key', label: 'Secret access key', required: true } + ], + createAdapter: createS3StorageAdapter +}) + +export const storageProviderRegistry = new StorageProviderRegistry([S3_STORAGE_PROVIDER]) diff --git a/src/app/integrations/storage/s3/adapter.ts b/src/app/integrations/storage/s3/adapter.ts new file mode 100644 index 000000000..4ac33d09b --- /dev/null +++ b/src/app/integrations/storage/s3/adapter.ts @@ -0,0 +1,254 @@ +import { isTauri } from '@/app/tauri/env' + +import { + NAMESPACE_MARKER_BODY, + STORAGE_DOCUMENTS_PREFIX, + STORAGE_NAMESPACE, + STORAGE_NAMESPACE_MARKER, + documentFigKey, + documentIdFromFigKey, + documentMetaKey, + documentThumbnailKey +} from '../namespace' +import type { + StorageAdapter, + StorageDocument, + StorageDocumentMetadata, + StorageProviderRuntime +} from '../types' +import { S3HttpError, deleteObject, getObject, headObject, listObjects, putObject } from './client' +import { + CloudCorsError, + ensureWebCorsOnBucket, + formatBrowserCorsHelpMessage, + isLikelyCorsOrNetworkError +} from './cors' +import type { S3CompatibleConfig, S3ConnectionResult } from './types' + +const ENDPOINT_FIELD = 'endpoint' +const BUCKET_FIELD = 'bucket' +const REGION_FIELD = 'region' +const ACCESS_KEY_FIELD = 'access-key-id' +const SECRET_KEY_FIELD = 'secret-access-key' + +function requiredPreference(runtime: StorageProviderRuntime, field: string): string { + const value = runtime.preferences[field]?.trim() + if (!value) throw new Error(`S3 ${field} is required`) + return value +} + +async function resolveConfig(runtime: StorageProviderRuntime): Promise { + const [accessKeyId, secretAccessKey] = await Promise.all([ + runtime.resolveCredential(ACCESS_KEY_FIELD), + runtime.resolveCredential(SECRET_KEY_FIELD) + ]) + if (!accessKeyId || !secretAccessKey) throw new Error('S3 credentials are required') + + const region = runtime.preferences[REGION_FIELD]?.trim() + return { + endpoint: requiredPreference(runtime, ENDPOINT_FIELD), + bucket: requiredPreference(runtime, BUCKET_FIELD), + accessKeyId, + secretAccessKey, + ...(region ? { region } : {}) + } +} + +function parseMetadata( + bytes: Uint8Array | null, + fallback: StorageDocumentMetadata +): { metadata: StorageDocumentMetadata; authoritative: boolean } { + if (!bytes) return { metadata: fallback, authoritative: false } + try { + const parsed = JSON.parse(new TextDecoder().decode(bytes)) as Partial + const name = typeof parsed.name === 'string' && parsed.name.trim() ? parsed.name : null + const updatedAt = + typeof parsed.updatedAt === 'string' && parsed.updatedAt ? parsed.updatedAt : null + return { + metadata: { + name: name ?? fallback.name, + updatedAt: updatedAt ?? fallback.updatedAt + }, + authoritative: name !== null && updatedAt !== null + } + } catch { + return { metadata: fallback, authoritative: false } + } +} + +function connectionErrorMessage(error: unknown, isCors: boolean): string { + if (isCors) return formatBrowserCorsHelpMessage() + return error instanceof Error ? error.message : String(error) +} + +async function ensureNamespace(config: S3CompatibleConfig): Promise { + if (await headObject(config, STORAGE_NAMESPACE_MARKER)) return + try { + await putObject(config, STORAGE_NAMESPACE_MARKER, NAMESPACE_MARKER_BODY, 'application/json') + } catch (error) { + if (error instanceof S3HttpError && (error.status === 403 || error.status === 401)) { + throw new Error('Cannot write to this bucket. Check access permissions and bucket name.') + } + throw error + } +} + +export interface S3StorageAdapter extends StorageAdapter { + testConnection(): Promise +} + +export function createS3StorageAdapter(runtime: StorageProviderRuntime): S3StorageAdapter { + return { + async testConnection() { + const config = await resolveConfig(runtime) + let cors = await ensureWebCorsOnBucket(config) + if (!cors.applied) console.warn('[Storage] Automatic PutBucketCors failed:', cors.error) + + try { + await ensureNamespace(config) + await listObjects(config, STORAGE_DOCUMENTS_PREFIX) + } catch (error) { + const isCors = + error instanceof CloudCorsError || (!isTauri() && isLikelyCorsOrNetworkError(error)) + return { + ok: false, + message: connectionErrorMessage(error, isCors), + corsApplied: cors.applied, + isCorsFailure: isCors, + corsError: cors.error + } + } + + if (!cors.applied) cors = await ensureWebCorsOnBucket(config) + return { + ok: true, + message: cors.applied + ? 'Connected. Bucket CORS was applied automatically for the web app.' + : 'Connected. Storage namespace is ready.', + corsApplied: cors.applied, + isCorsFailure: false, + corsError: null + } + }, + + async listDocuments() { + const config = await resolveConfig(runtime) + const objects = await listObjects(config, STORAGE_DOCUMENTS_PREFIX) + const entries = objects + .map((object) => { + const id = documentIdFromFigKey(object.key) + return id ? { id, lastModified: object.lastModified } : null + }) + .filter((entry): entry is { id: string; lastModified: string | null } => entry !== null) + + const documents = await Promise.all( + entries.map(async ({ id, lastModified }) => { + const fallback = { + name: id, + updatedAt: lastModified ?? new Date(0).toISOString() + } + const metadataBytes = await getObject(config, documentMetaKey(id)).catch( + (error: unknown) => { + console.warn('[Storage] Document metadata fetch failed:', id, error) + return null + } + ) + const { metadata, authoritative } = parseMetadata(metadataBytes, fallback) + return { + id, + ...metadata, + metadataAuthoritative: authoritative + } satisfies StorageDocument + }) + ) + return documents.sort((first, second) => second.updatedAt.localeCompare(first.updatedAt)) + }, + + async getDocument(id, onProgress) { + const config = await resolveConfig(runtime) + const bytes = await getObject( + config, + documentFigKey(id), + onProgress + ? (progress) => + onProgress({ + transferredBytes: progress.receivedBytes, + totalBytes: progress.totalBytes + }) + : undefined + ) + if (!bytes) throw new Error(`Document not found: ${id}`) + return bytes + }, + + async putDocument(id, bytes, metadata, onProgress) { + const config = await resolveConfig(runtime) + await putObject( + config, + documentFigKey(id), + bytes, + 'application/octet-stream', + onProgress + ? (progress) => + onProgress({ + transferredBytes: progress.sentBytes, + totalBytes: progress.totalBytes + }) + : undefined + ) + await putObject( + config, + documentMetaKey(id), + JSON.stringify({ + name: metadata.name, + updatedAt: metadata.updatedAt || new Date().toISOString() + }), + 'application/json' + ) + }, + + async getDocumentMetadata(id) { + const config = await resolveConfig(runtime) + const bytes = await getObject(config, documentMetaKey(id)) + if (!bytes) return null + const parsed = parseMetadata(bytes, { + name: id, + updatedAt: new Date(0).toISOString() + }) + return parsed.authoritative ? parsed.metadata : null + }, + + async deleteDocument(id) { + const config = await resolveConfig(runtime) + const results = await Promise.allSettled([ + deleteObject(config, documentFigKey(id)), + deleteObject(config, documentMetaKey(id)), + deleteObject(config, documentThumbnailKey(id)) + ]) + const failure = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failure) throw failure.reason + }, + + async getUsage() { + const config = await resolveConfig(runtime) + const objects = await listObjects(config, `${STORAGE_NAMESPACE}/`) + return { + bytesUsed: objects.reduce((total, object) => total + (object.size ?? 0), 0), + objectCount: objects.length, + documentCount: objects.filter((object) => documentIdFromFigKey(object.key)).length + } + }, + + async putThumbnail(id, bytes) { + const config = await resolveConfig(runtime) + await putObject(config, documentThumbnailKey(id), bytes, 'image/jpeg') + }, + + async getThumbnail(id) { + const config = await resolveConfig(runtime) + return getObject(config, documentThumbnailKey(id)) + } + } +} diff --git a/src/app/integrations/storage/s3/client.ts b/src/app/integrations/storage/s3/client.ts new file mode 100644 index 000000000..69c7faaef --- /dev/null +++ b/src/app/integrations/storage/s3/client.ts @@ -0,0 +1,324 @@ +import { AwsClient } from 'aws4fetch' + +import { storageFetch } from '@/app/integrations/storage/s3/fetch' +import { inferS3Region } from '@/app/integrations/storage/s3/region' +import type { S3CompatibleConfig } from '@/app/integrations/storage/s3/types' + +export function resolveS3Region(config: S3CompatibleConfig): string { + const explicit = config.region?.trim() + if (explicit) return explicit + return inferS3Region(config.endpoint) +} + +export class S3HttpError extends Error { + readonly status: number + readonly code: string | null + + constructor(status: number, message: string, code: string | null = null) { + super(message) + this.name = 'S3HttpError' + this.status = status + this.code = code + } +} + +export function normalizeEndpoint(endpoint: string): string { + const trimmed = endpoint.trim().replace(/\/+$/, '') + if (!trimmed) throw new Error('S3 endpoint is required') + if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed + return `https://${trimmed}` +} + +/** Alias for CORS module import clarity. */ +export const normalizeEndpointForCors = normalizeEndpoint + +/** Path-style object URL: {endpoint}/{bucket}/{key} — works with B2, MinIO, R2, AWS. */ +export function objectUrl(config: S3CompatibleConfig, key: string): string { + const base = normalizeEndpoint(config.endpoint) + const encodedKey = key + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/') + return `${base}/${encodeURIComponent(config.bucket)}/${encodedKey}` +} + +export function createAwsClient(config: S3CompatibleConfig): AwsClient { + return new AwsClient({ + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + region: resolveS3Region(config), + service: 's3' + }) +} + +async function readErrorBody(res: Response): Promise<{ message: string; code: string | null }> { + const text = await res.text().catch(() => '') + const codeMatch = text.match(/([^<]+)<\/Code>/i) + const messageMatch = text.match(/([^<]+)<\/Message>/i) + const code = codeMatch?.[1] ?? null + const message = + messageMatch?.[1] ?? + (text.trim() ? text.trim().slice(0, 200) : `S3 request failed with status ${res.status}`) + return { message, code } +} + +/** + * Known-length body types that UAs can send with Content-Length. + * Prefer these over Request-wrapped streams — B2 rejects missing Content-Length (411) + * and some browsers hang on chunked S3 PUTs. + */ +function bodyByteLength(body: BodyInit | null | undefined): number | null { + if (body == null) return null + if (typeof body === 'string') return new TextEncoder().encode(body).byteLength + if (body instanceof ArrayBuffer) return body.byteLength + if (ArrayBuffer.isView(body)) return body.byteLength + if (typeof Blob !== 'undefined' && body instanceof Blob) return body.size + return null +} + +export type UploadProgress = { sentBytes: number; totalBytes: number | null } + +/** + * fetch() cannot observe upload progress — replay the signed request over + * XMLHttpRequest when a progress callback is attached (uploads only). + */ +function xhrSend( + url: string, + method: string, + headers: Headers, + body: BodyInit | undefined, + onUploadProgress: (progress: UploadProgress) => void +): Promise { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest() + xhr.open(method, url) + headers.forEach((value, key) => { + // Forbidden request headers are set by the browser itself + if (/^(content-length|host)$/i.test(key)) return + xhr.setRequestHeader(key, value) + }) + xhr.responseType = 'text' + xhr.upload.onprogress = (e) => { + onUploadProgress({ sentBytes: e.loaded, totalBytes: e.lengthComputable ? e.total : null }) + } + xhr.onload = () => resolve(new Response(xhr.responseText, { status: xhr.status })) + // Same shape the fetch path throws so CORS/network detection keeps working + xhr.onerror = () => reject(new TypeError('Failed to fetch')) + xhr.send(body as XMLHttpRequestBodyInit) + }) +} + +export async function s3Request( + config: S3CompatibleConfig, + url: string, + init: RequestInit = {}, + onUploadProgress?: (progress: UploadProgress) => void +): Promise { + const client = createAwsClient(config) + const length = bodyByteLength(init.body ?? null) + const headers = new Headers(init.headers) + // Never send cookies; avoids credentialed CORS mode. + // Set Content-Length when we know size. aws4fetch leaves it unsignable (correct for S3). + // Critical for Backblaze B2 large binary PUTs. + if (length != null && !headers.has('Content-Length')) { + headers.set('Content-Length', String(length)) + } + // Sign with aws4fetch, then re-issue with url+init so the body keeps a known length. + // Passing only the signed Request object can drop Content-Length on some runtimes. + const signed = await client.sign(url, { + ...init, + headers, + credentials: 'omit' + }) + let res: Response + try { + if (onUploadProgress && typeof XMLHttpRequest !== 'undefined') { + res = await xhrSend( + signed.url, + signed.method, + signed.headers, + init.body ?? undefined, + onUploadProgress + ) + } else { + res = await storageFetch(signed.url, { + method: signed.method, + headers: signed.headers, + body: init.body ?? undefined, + credentials: 'omit' + }) + } + } catch (error) { + // Re-export as a typed error so UI can detect CORS/network blocks. + const { CloudCorsError, isLikelyCorsOrNetworkError, formatBrowserCorsHelpMessage } = + await import('@/app/integrations/storage/s3/cors') + if (isLikelyCorsOrNetworkError(error)) { + throw new CloudCorsError(formatBrowserCorsHelpMessage()) + } + throw error + } + if (res.ok || res.status === 404) return res + const { message, code } = await readErrorBody(res) + throw new S3HttpError(res.status, message, code) +} + +export async function headObject(config: S3CompatibleConfig, key: string): Promise { + const res = await s3Request(config, objectUrl(config, key), { method: 'HEAD' }) + if (res.status === 404) return false + return true +} + +export async function putObject( + config: S3CompatibleConfig, + key: string, + body: Uint8Array | string, + contentType: string, + onUploadProgress?: (progress: UploadProgress) => void +): Promise { + const bytes = typeof body === 'string' ? new TextEncoder().encode(body) : body + // Exact ArrayBuffer so fetch/UA can set Content-Length (required by B2 for large PUTs). + const payload = bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength + ) as ArrayBuffer + const res = await s3Request( + config, + objectUrl(config, key), + { + method: 'PUT', + headers: { + 'Content-Type': contentType + }, + body: payload + }, + onUploadProgress + ) + if (!res.ok) { + throw new S3HttpError(res.status, `Failed to upload ${key}`) + } +} + +export type DownloadProgress = { receivedBytes: number; totalBytes: number | null } + +export async function getObject( + config: S3CompatibleConfig, + key: string, + onProgress?: (progress: DownloadProgress) => void +): Promise { + const res = await s3Request(config, objectUrl(config, key), { method: 'GET' }) + if (res.status === 404) return null + if (!onProgress || !res.body) { + return new Uint8Array(await res.arrayBuffer()) + } + + // Stream so large figs can report download progress + const contentLength = Number(res.headers.get('content-length')) + const totalBytes = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : null + const reader = res.body.getReader() + const chunks: Uint8Array[] = [] + let receivedBytes = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) break + chunks.push(value) + receivedBytes += value.byteLength + onProgress({ receivedBytes, totalBytes }) + } + const out = new Uint8Array(receivedBytes) + let offset = 0 + for (const chunk of chunks) { + out.set(chunk, offset) + offset += chunk.byteLength + } + return out +} + +export async function deleteObject(config: S3CompatibleConfig, key: string): Promise { + const res = await s3Request(config, objectUrl(config, key), { method: 'DELETE' }) + if (!res.ok && res.status !== 404) { + throw new S3HttpError(res.status, `Failed to delete ${key}`) + } +} + +export type ListedObject = { + key: string + lastModified: string | null + size: number | null +} + +export type ListObjectsPage = { + objects: ListedObject[] + isTruncated: boolean + nextContinuationToken: string | null +} + +/** Parse ListObjectsV2 XML into key entries. Pure for unit tests. */ +export function parseListObjectsV2Xml(xml: string): ListedObject[] { + return parseListObjectsV2Page(xml).objects +} + +/** Parse ListObjectsV2 XML including pagination fields. */ +export function parseListObjectsV2Page(xml: string): ListObjectsPage { + const contents = [...xml.matchAll(/([\s\S]*?)<\/Contents>/gi)] + const items: ListedObject[] = [] + for (const match of contents) { + const block = match[1] ?? '' + const key = block.match(/([^<]*)<\/Key>/i)?.[1] + if (!key) continue + const lastModified = block.match(/([^<]*)<\/LastModified>/i)?.[1] ?? null + const sizeRaw = block.match(/([^<]*)<\/Size>/i)?.[1] + const size = sizeRaw != null && sizeRaw !== '' ? Number(sizeRaw) : null + items.push({ + key: decodeXmlEntities(key), + lastModified, + size: Number.isFinite(size) ? size : null + }) + } + const truncatedRaw = xml.match(/([^<]*)<\/IsTruncated>/i)?.[1] + const isTruncated = truncatedRaw?.trim().toLowerCase() === 'true' + const tokenRaw = xml.match(/([^<]*)<\/NextContinuationToken>/i)?.[1] + return { + objects: items, + isTruncated, + nextContinuationToken: tokenRaw ? decodeXmlEntities(tokenRaw) : null + } +} + +function decodeXmlEntities(value: string): string { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') +} + +export async function listObjects( + config: S3CompatibleConfig, + prefix: string +): Promise { + const base = normalizeEndpoint(config.endpoint) + const all: ListedObject[] = [] + let continuationToken: string | null = null + + for (let page = 0; page < 50; page++) { + const params = new URLSearchParams({ + 'list-type': '2', + prefix, + 'max-keys': '1000' + }) + if (continuationToken) params.set('continuation-token', continuationToken) + const url = `${base}/${encodeURIComponent(config.bucket)}?${params.toString()}` + const res = await s3Request(config, url, { method: 'GET' }) + if (!res.ok) { + throw new S3HttpError(res.status, 'Failed to list objects') + } + const xml = await res.text() + const parsed = parseListObjectsV2Page(xml) + all.push(...parsed.objects) + if (!parsed.isTruncated || !parsed.nextContinuationToken) break + continuationToken = parsed.nextContinuationToken + } + + return all +} diff --git a/src/app/integrations/storage/s3/cors.ts b/src/app/integrations/storage/s3/cors.ts new file mode 100644 index 000000000..1de21f735 --- /dev/null +++ b/src/app/integrations/storage/s3/cors.ts @@ -0,0 +1,198 @@ +import { + createAwsClient, + normalizeEndpointForCors, + S3HttpError +} from '@/app/integrations/storage/s3/client' +import { storageFetch } from '@/app/integrations/storage/s3/fetch' +import type { S3CompatibleConfig } from '@/app/integrations/storage/s3/types' +import { isTauri } from '@/app/tauri/env' +import { IS_BROWSER, WEB_APP_ORIGIN } from '@/constants' + +/** Origins OpenPencil may run from when calling S3 from the browser. */ +export const CLOUD_CORS_STATIC_ORIGINS = [ + // Exact production origin — providers without partial-wildcard support + // (e.g. R2) need it verbatim even when CORS is configured from dev. + WEB_APP_ORIGIN, + // Wildcards: any openpencil.dev subdomain (staging, demo, …), Cloudflare + // Pages PR previews, and any local dev port. S3/B2 allow one '*' per origin. + // collectCloudCorsOrigins() also appends the current origin, so strict + // providers still get an exact match for wherever the app is running when + // CORS is applied. + 'https://*.openpencil.dev', + 'https://*.openpencil-app.pages.dev', + 'http://localhost:*', + 'http://127.0.0.1:*' +] as const + +export function collectCloudCorsOrigins(extra?: string | null): string[] { + const set = new Set(CLOUD_CORS_STATIC_ORIGINS) + if (extra?.trim()) set.add(extra.trim().replace(/\/+$/, '')) + if (IS_BROWSER && window.location.origin) { + set.add(window.location.origin) + } + return [...set].filter(Boolean).sort() +} + +function escapeXml(value: string): string { + return value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +/** S3 PutBucketCors XML body (AWS + B2 S3-compatible). */ +export function buildCorsConfigurationXml(origins: string[]): string { + const originTags = origins + .map((origin) => ` ${escapeXml(origin)}`) + .join('\n') + return ` + + +${originTags} + GET + PUT + POST + DELETE + HEAD + * + ETag + x-amz-request-id + x-amz-id-2 + x-amz-version-id + 3600 + + +` +} + +/** AWS console / CLI JSON CORS document (copy-paste friendly). */ +export function buildCorsConfigurationJson(origins: string[]): string { + return JSON.stringify( + [ + { + AllowedHeaders: ['*'], + AllowedMethods: ['GET', 'PUT', 'POST', 'DELETE', 'HEAD'], + AllowedOrigins: origins, + ExposeHeaders: ['ETag', 'x-amz-request-id', 'x-amz-id-2', 'x-amz-version-id'], + MaxAgeSeconds: 3600 + } + ], + null, + 2 + ) +} + +export class CloudCorsError extends Error { + readonly kind = 'cors' as const + + constructor(message: string) { + super(message) + this.name = 'CloudCorsError' + } +} + +/** Best-effort detection of browser CORS / network blocks (preflight failures). */ +export function isLikelyCorsOrNetworkError(error: unknown): boolean { + if (error instanceof CloudCorsError) return true + if (error instanceof TypeError) return true + if (!(error instanceof Error)) return false + const msg = error.message.toLowerCase() + return ( + msg.includes('failed to fetch') || + msg.includes('networkerror') || + msg.includes('network request failed') || + msg.includes('load failed') || + msg.includes('cors') || + msg.includes('access-control') || + msg.includes('blocked by cors') + ) +} + +export function formatBrowserCorsHelpMessage(): string { + return ( + 'CORS issue: the browser blocked access to your bucket. ' + + 'OpenPencil tried to set CORS automatically but could not from the web app ' + + '(the bucket must already allow this site, or use the desktop app once). ' + + 'Click “Copy CORS JSON”, paste it into your bucket CORS settings, wait ~1 minute, then try again.' + ) +} + +/** + * Apply recommended CORS via S3 PutBucketCors — same operation as AWS CLI put-bucket-cors. + * - Desktop: always works (no browser CORS on the request itself). + * - Web: only works if the bucket already allows this origin for PUT, or CORS was set externally. + */ +export async function putBucketCors( + config: S3CompatibleConfig, + origins: string[] = collectCloudCorsOrigins() +): Promise { + const base = normalizeEndpointForCors(config.endpoint) + const url = `${base}/${encodeURIComponent(config.bucket)}?cors` + const body = buildCorsConfigurationXml(origins) + const client = createAwsClient(config) + let signed: Request + try { + signed = await client.sign(url, { + method: 'PUT', + headers: { + 'Content-Type': 'application/xml' + }, + body, + credentials: 'omit' + }) + } catch (error) { + throw error instanceof Error ? error : new Error(String(error)) + } + + let res: Response + try { + res = await storageFetch(signed) + } catch (error) { + if (isLikelyCorsOrNetworkError(error)) { + throw new CloudCorsError( + isTauri() + ? 'Network error while applying bucket CORS.' + : 'Could not apply CORS from the browser (preflight blocked). Use Copy CORS JSON or the desktop app.' + ) + } + throw error instanceof Error ? error : new Error(String(error)) + } + + if (!res.ok) { + const text = await res.text().catch(() => '') + if (res.status === 403 || res.status === 401) { + throw new S3HttpError( + res.status, + 'Access key cannot update bucket CORS. Grant bucket write / PutBucketCors permission.' + ) + } + throw new S3HttpError( + res.status, + text.trim().slice(0, 200) || `PutBucketCors failed with status ${res.status}` + ) + } +} + +export type EnsureCorsResult = { + applied: boolean + error: string | null +} + +/** + * Always attempt PutBucketCors (automatic CORS setup for the web app origins). + * Returns whether it worked; never throws. + */ +export async function ensureWebCorsOnBucket(config: S3CompatibleConfig): Promise { + const origins = collectCloudCorsOrigins() + try { + await putBucketCors(config, origins) + return { applied: true, error: null } + } catch (error) { + return { + applied: false, + error: error instanceof Error ? error.message : String(error) + } + } +} diff --git a/src/app/integrations/storage/s3/fetch.ts b/src/app/integrations/storage/s3/fetch.ts new file mode 100644 index 000000000..912b14aae --- /dev/null +++ b/src/app/integrations/storage/s3/fetch.ts @@ -0,0 +1,49 @@ +import { isTauri } from '@/app/tauri/env' + +/** Avoid hung “Test connection” when CORS/network never resolves. */ +const STORAGE_FETCH_TIMEOUT_MS = 20_000 + +function withTimeoutSignal(init?: RequestInit): { + signal: AbortSignal + cleanup: () => void +} { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), STORAGE_FETCH_TIMEOUT_MS) + const external = init?.signal + if (external) { + if (external.aborted) controller.abort() + else external.addEventListener('abort', () => controller.abort(), { once: true }) + } + return { + signal: controller.signal, + cleanup: () => clearTimeout(timer) + } +} + +/** Prefer Tauri HTTP bridge on desktop to avoid bucket CORS requirements. */ +export async function storageFetch( + input: RequestInfo | URL, + init?: RequestInit +): Promise { + const { signal, cleanup } = withTimeoutSignal(init) + try { + if (isTauri()) { + const { tauriFetch } = await import('@/app/tauri/http') + return await tauriFetch(input, { ...init, signal }) + } + // Request bodies are owned by the Request; re-wrap so we can attach a timeout signal. + if (input instanceof Request) { + return await fetch(new Request(input, { signal })) + } + return await fetch(input, { ...init, signal }) + } catch (error) { + if (signal.aborted) { + throw new Error( + 'Storage request timed out. Check the endpoint URL, network, and bucket CORS settings.' + ) + } + throw error + } finally { + cleanup() + } +} diff --git a/src/app/integrations/storage/s3/region.ts b/src/app/integrations/storage/s3/region.ts new file mode 100644 index 000000000..611f8a4c0 --- /dev/null +++ b/src/app/integrations/storage/s3/region.ts @@ -0,0 +1,41 @@ +/** + * Infer SigV4 signing region from a common S3-compatible endpoint host. + * Region is not shown in the UI — most providers encode it in the URL. + */ +export function inferS3Region(endpoint: string, fallback = 'us-east-1'): string { + const trimmed = endpoint.trim() + if (!trimmed) return fallback + + let host: string + try { + const withProto = + trimmed.startsWith('http://') || trimmed.startsWith('https://') + ? trimmed + : `https://${trimmed}` + host = new URL(withProto).hostname.toLowerCase() + } catch { + return fallback + } + + // Backblaze B2: s3.eu-central-003.backblazeb2.com + const b2 = host.match(/^s3\.([a-z0-9-]+)\.backblazeb2\.com$/) + if (b2?.[1]) return b2[1] + + // AWS path-style: s3.eu-west-1.amazonaws.com / s3-eu-west-1.amazonaws.com + const awsPath = host.match(/^s3[.-]([a-z0-9-]+)\.amazonaws\.com$/) + if (awsPath?.[1] && awsPath[1] !== 'dualstack' && awsPath[1] !== 'control') { + return awsPath[1] + } + + // AWS virtual-hosted: bucket.s3.eu-west-1.amazonaws.com + const awsVirtual = host.match(/\.s3[.-]([a-z0-9-]+)\.amazonaws\.com$/) + if (awsVirtual?.[1] && awsVirtual[1] !== 'dualstack') return awsVirtual[1] + + // Cloudflare R2 + if (host.endsWith('.r2.cloudflarestorage.com') || host === 'r2.cloudflarestorage.com') { + return 'auto' + } + + // MinIO / custom: no region in host — SigV4 still wants a string. + return fallback +} diff --git a/src/app/integrations/storage/s3/types.ts b/src/app/integrations/storage/s3/types.ts new file mode 100644 index 000000000..98afafd09 --- /dev/null +++ b/src/app/integrations/storage/s3/types.ts @@ -0,0 +1,15 @@ +export type S3CompatibleConfig = { + endpoint: string + bucket: string + accessKeyId: string + secretAccessKey: string + region?: string +} + +export type S3ConnectionResult = { + ok: boolean + message: string + corsApplied: boolean + isCorsFailure: boolean + corsError: string | null +} diff --git a/src/app/integrations/storage/types.ts b/src/app/integrations/storage/types.ts index 5d990102c..da97ba73c 100644 --- a/src/app/integrations/storage/types.ts +++ b/src/app/integrations/storage/types.ts @@ -16,6 +16,7 @@ export type StorageDocumentMetadata = { export type StorageDocument = StorageDocumentMetadata & { id: string thumbnailUrl?: string | null + metadataAuthoritative?: boolean } export type StorageUsage = { @@ -43,6 +44,7 @@ export interface StorageAdapter { onProgress?: (progress: StorageTransferProgress) => void ): Promise deleteDocument(id: string): Promise + getDocumentMetadata?(id: string): Promise getUsage(): Promise getThumbnail?(id: string): Promise putThumbnail?(id: string, bytes: Uint8Array): Promise diff --git a/tests/engine/app/integrations/storage/cors.test.ts b/tests/engine/app/integrations/storage/cors.test.ts new file mode 100644 index 000000000..b839543fc --- /dev/null +++ b/tests/engine/app/integrations/storage/cors.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from 'bun:test' + +import { + buildCorsConfigurationJson, + buildCorsConfigurationXml, + collectCloudCorsOrigins, + isLikelyCorsOrNetworkError +} from '@/app/integrations/storage/s3/cors' +import { WEB_APP_ORIGIN } from '@/constants' + +describe('cloud S3 CORS helpers', () => { + test('builds XML with required methods and wildcard headers', () => { + const xml = buildCorsConfigurationXml(['https://app.openpencil.dev', 'http://localhost:1420']) + expect(xml).toContain('https://app.openpencil.dev') + expect(xml).toContain('http://localhost:1420') + expect(xml).toContain('GET') + expect(xml).toContain('PUT') + expect(xml).toContain('HEAD') + expect(xml).toContain('DELETE') + expect(xml).toContain('*') + expect(xml).toContain('ETag') + }) + + test('escapes XML special characters in origins', () => { + const xml = buildCorsConfigurationXml(['https://example.com/a&b']) + expect(xml).toContain('https://example.com/a&b') + }) + + test('builds AWS console JSON', () => { + const json = JSON.parse(buildCorsConfigurationJson(['https://app.openpencil.dev'])) as Array<{ + AllowedOrigins: string[] + AllowedMethods: string[] + AllowedHeaders: string[] + }> + expect(json).toHaveLength(1) + expect(json[0]?.AllowedOrigins).toContain('https://app.openpencil.dev') + expect(json[0]?.AllowedMethods).toEqual( + expect.arrayContaining(['GET', 'PUT', 'POST', 'DELETE', 'HEAD']) + ) + expect(json[0]?.AllowedHeaders).toEqual(['*']) + }) + + test('collects static web and localhost origins', () => { + const origins = collectCloudCorsOrigins() + expect(origins).toContain(WEB_APP_ORIGIN) + expect(origins).toContain('https://*.openpencil.dev') + expect(origins).toContain('https://*.openpencil-app.pages.dev') + expect(origins).toContain('http://localhost:*') + expect(origins).toContain('http://127.0.0.1:*') + }) + + test('detects typical browser CORS/network failures', () => { + expect(isLikelyCorsOrNetworkError(new TypeError('Failed to fetch'))).toBe(true) + expect(isLikelyCorsOrNetworkError(new Error('blocked by CORS policy'))).toBe(true) + expect(isLikelyCorsOrNetworkError(new Error('Access key invalid'))).toBe(false) + }) +}) diff --git a/tests/engine/app/integrations/storage/list-objects.test.ts b/tests/engine/app/integrations/storage/list-objects.test.ts new file mode 100644 index 000000000..315f38519 --- /dev/null +++ b/tests/engine/app/integrations/storage/list-objects.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from 'bun:test' + +import { documentIdFromFigKey } from '@/app/integrations/storage/namespace' +import { parseListObjectsV2Xml } from '@/app/integrations/storage/s3/client' + +describe('parseListObjectsV2Xml', () => { + test('extracts keys and ignores objects outside canvas fig pattern when filtered', () => { + const xml = ` + + + open_pencil_storage/canvases/a1.fig + 2026-01-02T03:04:05.000Z + 12 + + + open_pencil_storage/canvases/a1.meta.json + 2026-01-02T03:04:06.000Z + 40 + + + other-app/file.bin + 2026-01-01T00:00:00.000Z + 1 + +` + + const listed = parseListObjectsV2Xml(xml) + expect(listed).toHaveLength(3) + expect(listed[0]?.key).toBe('open_pencil_storage/canvases/a1.fig') + expect(listed[0]?.lastModified).toBe('2026-01-02T03:04:05.000Z') + expect(listed[0]?.size).toBe(12) + + const canvasIds = listed + .map((object) => documentIdFromFigKey(object.key)) + .filter((id): id is string => id != null) + expect(canvasIds).toEqual(['a1']) + }) + + test('decodes basic XML entities in keys', () => { + const xml = `open_pencil_storage/canvases/a&b.fig1` + const listed = parseListObjectsV2Xml(xml) + expect(listed[0]?.key).toBe('open_pencil_storage/canvases/a&b.fig') + }) +}) diff --git a/tests/engine/app/integrations/storage/namespace.test.ts b/tests/engine/app/integrations/storage/namespace.test.ts new file mode 100644 index 000000000..e3f00fd4c --- /dev/null +++ b/tests/engine/app/integrations/storage/namespace.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from 'bun:test' + +import { + STORAGE_DOCUMENTS_PREFIX, + STORAGE_NAMESPACE, + STORAGE_NAMESPACE_MARKER, + documentFigKey, + documentIdFromFigKey, + documentMetaKey, + documentThumbnailKey +} from '@/app/integrations/storage/namespace' + +describe('storage namespace', () => { + test('uses the fixed open_pencil_storage prefix', () => { + expect(STORAGE_NAMESPACE).toBe('open_pencil_storage') + expect(STORAGE_NAMESPACE_MARKER.startsWith(`${STORAGE_NAMESPACE}/`)).toBe(true) + expect(STORAGE_DOCUMENTS_PREFIX).toBe('open_pencil_storage/canvases/') + }) + + test('builds document object keys inside the namespace', () => { + const id = 'abc-123' + expect(documentFigKey(id)).toBe('open_pencil_storage/canvases/abc-123.fig') + expect(documentMetaKey(id)).toBe('open_pencil_storage/canvases/abc-123.meta.json') + expect(documentThumbnailKey(id)).toBe('open_pencil_storage/canvases/abc-123.thumb.jpg') + }) + + test('parses document IDs from fig keys and ignores foreign keys', () => { + expect(documentIdFromFigKey('open_pencil_storage/canvases/uuid-1.fig')).toBe('uuid-1') + expect(documentIdFromFigKey('other_prefix/canvases/uuid-1.fig')).toBeNull() + expect(documentIdFromFigKey('open_pencil_storage/canvases/nested/uuid-1.fig')).toBeNull() + expect(documentIdFromFigKey('open_pencil_storage/canvases/uuid-1.meta.json')).toBeNull() + }) +}) diff --git a/tests/engine/app/integrations/storage/region.test.ts b/tests/engine/app/integrations/storage/region.test.ts new file mode 100644 index 000000000..18ebd339a --- /dev/null +++ b/tests/engine/app/integrations/storage/region.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from 'bun:test' + +import { inferS3Region } from '@/app/integrations/storage/s3/region' + +describe('inferS3Region', () => { + test('parses Backblaze B2 endpoints', () => { + expect(inferS3Region('https://s3.eu-central-003.backblazeb2.com')).toBe('eu-central-003') + expect(inferS3Region('s3.us-west-004.backblazeb2.com')).toBe('us-west-004') + }) + + test('parses AWS path-style endpoints', () => { + expect(inferS3Region('https://s3.eu-west-1.amazonaws.com')).toBe('eu-west-1') + expect(inferS3Region('https://s3-us-east-1.amazonaws.com')).toBe('us-east-1') + }) + + test('uses auto for Cloudflare R2', () => { + expect(inferS3Region('https://abc123.r2.cloudflarestorage.com')).toBe('auto') + }) + + test('falls back for custom MinIO hosts', () => { + expect(inferS3Region('https://minio.example.com')).toBe('us-east-1') + }) +}) diff --git a/tests/engine/app/integrations/storage-registry.test.ts b/tests/engine/app/integrations/storage/registry.test.ts similarity index 89% rename from tests/engine/app/integrations/storage-registry.test.ts rename to tests/engine/app/integrations/storage/registry.test.ts index dc32c7e52..f67c30745 100644 --- a/tests/engine/app/integrations/storage-registry.test.ts +++ b/tests/engine/app/integrations/storage/registry.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test' import { defineStorageProvider, + storageProviderRegistry, StorageProviderRegistry, type StorageAdapter, type StorageProviderRuntime @@ -50,6 +51,20 @@ function testProvider() { } describe('storage provider registry', () => { + test('registers S3 preferences separately from credential fields', () => { + const provider = storageProviderRegistry.get('s3-compatible') + + expect(provider.preferenceFields.map((field) => field.id)).toEqual([ + 'endpoint', + 'bucket', + 'region' + ]) + expect(provider.credentialFields.map((field) => field.id)).toEqual([ + 'access-key-id', + 'secret-access-key' + ]) + }) + test('lists provider schemas without resolving credentials', () => { let resolutionCount = 0 const credentials: CredentialResolver = {