la-rose/scripts/fetch-images.ts

import { readFileSync } from 'node:fs';
import { mkdir, readFile as readFileAsync, stat, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

/**
 * Fetch the full PDP gallery for every product in db/seed/catalog.tsv and store
 * the originals under public/images/products/<slug>/<n>.jpg (docs/PLAN.md §11).
 *
 * Source recipe (verified against live la-rose.by PDPs):
 *   • GET https://la-rose.by/product/<slug>/
 *   • gallery = anchors a.product_gallery-images-main-el inside .product_gallery-images-main
 *   • full-size url = the anchor href (already `.900.jpg`); relative → prefix https://la-rose.by
 *   • order = data-position (0-based) → position = data-position + 1 (0 = the main photo)
 *   • ignore img[src] (`.650x0.jpg`), img[data-src] and srcset — originals only (I3)
 *
 * Idempotent: a file whose source url is unchanged is skipped, and every run
 * rewrites db/seed/images.tsv (slug ⇥ position ⇥ source_url ⇥ alt) from the
 * source. Exits non-zero when any gallery fails to parse or any download fails,
 * and always prints the explicit list of products whose gallery was not parsed.
 */

const ORIGIN = 'https://la-rose.by';
const USER_AGENT =
  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36 la-rose-seed/1.0';
const CONCURRENCY = 6;
const REQUEST_TIMEOUT_MS = 30_000;

const CATALOG_TSV = fileURLToPath(new URL('../db/seed/catalog.tsv', import.meta.url));
const IMAGES_TSV = fileURLToPath(new URL('../db/seed/images.tsv', import.meta.url));
const PUBLIC_DIR = fileURLToPath(new URL('../public', import.meta.url));

interface GalleryImage {
  position: number;
  sourceUrl: string;
  alt: string;
}

interface ImageRow extends GalleryImage {
  slug: string;
}

/** Slugs in catalog order — the second tab-separated column of catalog.tsv. */
function readSlugs(): string[] {
  const text = readFileSync(CATALOG_TSV, 'utf8');
  const slugs: string[] = [];
  for (const line of text.split(/\r?\n/)) {
    if (!line || line.startsWith('#')) continue;
    const slug = line.split('\t')[1]?.trim();
    if (slug) slugs.push(slug);
  }
  if (slugs.length === 0) throw new Error(`no slugs parsed from ${CATALOG_TSV}`);
  return slugs;
}

const ENTITIES: Record<string, string> = {
  '&quot;': '"',
  '&#34;': '"',
  '&#39;': "'",
  '&apos;': "'",
  '&amp;': '&',
  '&lt;': '<',
  '&gt;': '>',
  '&nbsp;': ' ',
};

function decodeEntities(value: string): string {
  return value.replace(/&(?:quot|amp|lt|gt|apos|nbsp|#34|#39);/gi, (match) => ENTITIES[match.toLowerCase()] ?? match);
}

function attr(attributes: string, name: string): string | null {
  const match = attributes.match(new RegExp(`\\b${name}\\s*=\\s*"([^"]*)"`, 'i'));
  return match ? match[1] : null;
}

/** Parse the main gallery anchors → ordered images (position is 1-based). */
function parseGallery(html: string): GalleryImage[] {
  const images: GalleryImage[] = [];
  const seen = new Set<string>();
  const anchor = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
  let match: RegExpExecArray | null;
  while ((match = anchor.exec(html)) !== null) {
    const attributes = match[1];
    const classes = (attr(attributes, 'class') ?? '').split(/\s+/);
    if (!classes.includes('product_gallery-images-main-el')) continue;
    const href = attr(attributes, 'href');
    if (!href) continue;

    const positionAttr = attr(attributes, 'data-position');
    const dataPosition =
      positionAttr !== null && /^\d+$/.test(positionAttr.trim()) ? Number(positionAttr.trim()) : images.length;

    const imgTag = match[2].match(/<img\b[^>]*>/i)?.[0] ?? '';
    const alt = decodeEntities(attr(imgTag, 'alt') ?? attr(imgTag, 'title') ?? '')
      .replace(/\s+/g, ' ')
      .trim();

    const sourceUrl = href.startsWith('http') ? href : `${ORIGIN}${href.startsWith('/') ? '' : '/'}${href}`;
    if (seen.has(sourceUrl)) continue;
    seen.add(sourceUrl);
    images.push({ position: dataPosition + 1, sourceUrl, alt });
  }
  images.sort((a, b) => a.position - b.position);
  return images;
}

async function get(url: string, accept: string): Promise<Response> {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
  try {
    const response = await fetch(url, {
      headers: { 'user-agent': USER_AGENT, accept },
      redirect: 'follow',
      signal: controller.signal,
    });
    if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
    return response;
  } finally {
    clearTimeout(timer);
  }
}

/** Run `task` over `items` with a bounded number of concurrent workers. */
async function mapLimit<T>(items: readonly T[], limit: number, task: (item: T) => Promise<void>): Promise<void> {
  let cursor = 0;
  const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
    while (cursor < items.length) {
      await task(items[cursor++]);
    }
  });
  await Promise.all(workers);
}

function fileFor(slug: string, position: number): string {
  return join(PUBLIC_DIR, 'images', 'products', slug, `${position}.jpg`);
}

/** Previous index `slug\tposition` → source url, so unchanged files can be skipped. */
async function readExistingRows(): Promise<Map<string, string>> {
  const existing = new Map<string, string>();
  try {
    const text = await readFileAsync(IMAGES_TSV, 'utf8');
    for (const line of text.split(/\r?\n/)) {
      if (!line || line.startsWith('#')) continue;
      const [slug, position, sourceUrl] = line.split('\t');
      if (slug && position && sourceUrl) existing.set(`${slug}\t${position}`, sourceUrl);
    }
  } catch {
    // first run — no index yet
  }
  return existing;
}

async function download(sourceUrl: string, destination: string): Promise<number> {
  const response = await get(sourceUrl, 'image/jpeg,image/*;q=0.9,*/*;q=0.8');
  const bytes = Buffer.from(await response.arrayBuffer());
  await mkdir(dirname(destination), { recursive: true });
  await writeFile(destination, bytes);
  return bytes.length;
}

async function main(): Promise<void> {
  const slugs = readSlugs();
  const existing = await readExistingRows();
  const rows: ImageRow[] = [];
  const galleriesFailed: string[] = [];

  await mapLimit(slugs, CONCURRENCY, async (slug) => {
    try {
      const response = await get(`${ORIGIN}/product/${encodeURIComponent(slug)}/`, 'text/html,*/*;q=0.8');
      const gallery = parseGallery(await response.text());
      if (gallery.length === 0) {
        galleriesFailed.push(slug);
        console.error(`fetch-images: no gallery parsed for ${slug}`);
        return;
      }
      for (const image of gallery) rows.push({ slug, ...image });
    } catch (error) {
      galleriesFailed.push(slug);
      console.error(`fetch-images: gallery request failed for ${slug}: ${(error as Error).message}`);
    }
  });

  let downloaded = 0;
  let cached = 0;
  let bytes = 0;
  const downloadsFailed: string[] = [];

  await mapLimit(rows, CONCURRENCY, async (row) => {
    const destination = fileFor(row.slug, row.position);
    const previous = existing.get(`${row.slug}\t${row.position}`);
    try {
      const info = await stat(destination).catch(() => null);
      if (previous === row.sourceUrl && info && info.size > 0) {
        cached += 1;
        bytes += info.size;
        return;
      }
      bytes += await download(row.sourceUrl, destination);
      downloaded += 1;
    } catch (error) {
      downloadsFailed.push(`${row.slug}/${row.position} ${row.sourceUrl} — ${(error as Error).message}`);
    }
  });

  const order = new Map(slugs.map((slug, index) => [slug, index]));
  rows.sort((a, b) => (order.get(a.slug) ?? 0) - (order.get(b.slug) ?? 0) || a.position - b.position);

  const lines = [
    '# La Rose product gallery — docs/PLAN.md §11 (I1–I4).',
    '# Columns (tab separated): slug  position  source_url  alt',
    '# position = source data-position + 1 (1 = the main PDP photo); url in the DB = /images/products/<slug>/<position>.jpg',
    '# Generator: scripts/fetch-images.ts (`pnpm fetch:images`).',
    ...rows.map((row) => `${row.slug}\t${row.position}\t${row.sourceUrl}\t${row.alt}`),
  ];
  await writeFile(IMAGES_TSV, `${lines.join('\n')}\n`, 'utf8');

  console.log('fetch-images: summary');
  console.log(`  products               = ${slugs.length}`);
  console.log(`  galleries parsed       = ${slugs.length - galleriesFailed.length}/${slugs.length}`);
  console.log(`  image rows (images.tsv)= ${rows.length}`);
  console.log(`  files downloaded       = ${downloaded}`);
  console.log(`  files already present  = ${cached}`);
  console.log(`  gallery total size     = ${(bytes / (1024 * 1024)).toFixed(1)} MiB`);
  console.log(`  downloads failed       = ${downloadsFailed.length}`);
  console.log('  gallery not parsed:');
  console.log(galleriesFailed.length === 0 ? '    (none)' : galleriesFailed.map((slug) => `    - ${slug}`).join('\n'));
  if (downloadsFailed.length > 0) {
    console.log('  download failures:');
    console.log(downloadsFailed.map((entry) => `    - ${entry}`).join('\n'));
  }

  if (galleriesFailed.length > 0 || downloadsFailed.length > 0) {
    process.exitCode = 1;
  }
}

await main();