/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ // Diffs a locally rendered screenshot manifest against a base-commit manifest // and prints Markdown to stdout. Empty output (exit 0) means no reportable // changes. The base manifest is supplied as a file path — the caller is // responsible for fetching it (e.g. via curl from the screenshot service's // public /commits/// endpoint). // // Usage: // node build/lib/screenshotDiffReport.ts \ // // // Both before- and after-images are referenced via the public /images/ // endpoint of the screenshot service. After-images become available there // once the workflow's upload step has pushed them. Note: GitHub markdown // does not render `data:` URIs, so we cannot inline images as base64; for // runs that skip the upload (e.g. fork PRs) the after-image URL will 404 // until the service has the hash, which is acceptable since fork PRs do // not post a PR comment. import * as fs from 'fs'; import * as https from 'https'; import * as path from 'path'; import { fileURLToPath } from 'url'; import * as zlib from 'zlib'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const COMMENT_MARKER = ''; const EXPAND_FIRST_N = 5; const EXCLUDED_LABELS = new Set(['animated', 'flaky']); const MAX_BODY_BYTES = 300 * 1024; // --------------------------------------------------------------------------- // Pixel-level image comparison // --------------------------------------------------------------------------- const MAX_INSIGNIFICANT_PIXELS = 20; const MAX_INSIGNIFICANT_CHANNEL_DELTA = 2; interface RawImage { readonly width: number; readonly height: number; readonly data: Buffer; } function readPngPixels(buf: Buffer): RawImage { if (buf[0] !== 0x89 || buf[1] !== 0x50 || buf[2] !== 0x4E || buf[3] !== 0x47) { throw new Error('Not a PNG file'); } let offset = 8; let width = 0, height = 0, colorType = 0; const idatChunks: Buffer[] = []; while (offset < buf.length) { const length = buf.readUInt32BE(offset); const type = buf.toString('ascii', offset + 4, offset + 8); const data = buf.subarray(offset + 8, offset + 8 + length); if (type === 'IHDR') { width = data.readUInt32BE(0); height = data.readUInt32BE(4); if (data[8] !== 8) { throw new Error(`Unsupported bit depth: ${data[8]}`); } colorType = data[9]; if (colorType !== 2 && colorType !== 6) { throw new Error(`Unsupported color type: ${colorType}`); } } else if (type === 'IDAT') { idatChunks.push(data); } else if (type === 'IEND') { break; } offset += 12 + length; } const bpp = colorType === 6 ? 4 : 3; const raw = zlib.inflateSync(Buffer.concat(idatChunks)); const stride = width * bpp + 1; const pixels = Buffer.alloc(width * height * 4); const prevRow = Buffer.alloc(width * bpp); const currRow = Buffer.alloc(width * bpp); for (let y = 0; y < height; y++) { const filterType = raw[y * stride]; const rowData = raw.subarray(y * stride + 1, y * stride + 1 + width * bpp); for (let i = 0; i < width * bpp; i++) { const a = i >= bpp ? currRow[i - bpp] : 0; const b = prevRow[i]; const c = i >= bpp ? prevRow[i - bpp] : 0; let val = rowData[i]; switch (filterType) { case 0: break; case 1: val = (val + a) & 0xff; break; case 2: val = (val + b) & 0xff; break; case 3: val = (val + ((a + b) >> 1)) & 0xff; break; case 4: { const p = a + b - c; const pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); val = (val + (pa <= pb && pa <= pc ? a : pb <= pc ? b : c)) & 0xff; break; } } currRow[i] = val; } for (let x = 0; x < width; x++) { const pi = (y * width + x) * 4; pixels[pi] = currRow[x * bpp]; pixels[pi + 1] = currRow[x * bpp + 1]; pixels[pi + 2] = currRow[x * bpp + 2]; pixels[pi + 3] = bpp === 4 ? currRow[x * bpp + 3] : 255; } prevRow.set(currRow); } return { width, height, data: pixels }; } class ImageDiffResult { readonly changedPixelCount: number; readonly maxChannelDelta: number; constructor(changedPixelCount: number, maxChannelDelta: number) { this.changedPixelCount = changedPixelCount; this.maxChannelDelta = maxChannelDelta; } get isSignificant(): boolean { return this.changedPixelCount > MAX_INSIGNIFICANT_PIXELS || this.maxChannelDelta > MAX_INSIGNIFICANT_CHANNEL_DELTA; } toString(): string { if (this.changedPixelCount === 0) { return 'identical'; } return `${this.changedPixelCount} px changed, max \u0394${this.maxChannelDelta}${this.isSignificant ? '' : ' (insignificant)'}`; } } function diffImages(before: Buffer, after: Buffer): ImageDiffResult { const imgA = readPngPixels(before); const imgB = readPngPixels(after); if (imgA.width !== imgB.width || imgA.height !== imgB.height) { return new ImageDiffResult(imgA.width * imgA.height, 255); } let changedPixels = 0; let maxDelta = 0; for (let i = 0; i < imgA.data.length; i += 4) { const dr = Math.abs(imgA.data[i] - imgB.data[i]); const dg = Math.abs(imgA.data[i + 1] - imgB.data[i + 1]); const db = Math.abs(imgA.data[i + 2] - imgB.data[i + 2]); const da = Math.abs(imgA.data[i + 3] - imgB.data[i + 3]); if (dr || dg || db || da) { changedPixels++; maxDelta = Math.max(maxDelta, dr, dg, db, da); } } return new ImageDiffResult(changedPixels, maxDelta); } function fetchBuffer(url: string): Promise { return new Promise((resolve, reject) => { https.get(url, (res) => { if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { fetchBuffer(res.headers.location).then(resolve, reject); return; } if (!res.statusCode || res.statusCode !== 200) { reject(new Error(`HTTP ${res.statusCode} for ${url}`)); return; } const chunks: Buffer[] = []; res.on('data', (chunk: Buffer) => chunks.push(chunk)); res.on('end', () => resolve(Buffer.concat(chunks))); res.on('error', reject); }).on('error', reject); }); } async function computePixelDiffs( changed: readonly ChangedDiffEntry[], serviceUrl: string, localManifestDir: string, ): Promise> { const results = new Map(); await Promise.all(changed.map(async (entry) => { try { const beforeUrl = loadImageUrl(serviceUrl, entry.beforeHash); const [beforeBuf, afterBuf] = await Promise.all([ fetchBuffer(beforeUrl), fs.promises.readFile(path.resolve(localManifestDir, entry.afterPath)), ]); results.set(entry.fixtureId, diffImages(beforeBuf, afterBuf)); } catch (err) { console.error(` Warning: pixel diff failed for ${entry.fixtureId}: ${err}`); } })); return results; } interface ManifestEvent { readonly type?: string; readonly message?: string; readonly stack?: string; readonly phase?: string; readonly isError?: boolean; } interface LocalManifestFixture { readonly fixtureId: string; readonly imageHash?: string; readonly imagePath?: string; readonly labels?: readonly string[]; readonly hasError?: boolean; readonly events?: readonly ManifestEvent[]; } interface LocalManifest { readonly fixtures: readonly LocalManifestFixture[]; } interface BaseFixture { readonly fixtureId: string; readonly imageHash: string; readonly labels?: readonly string[]; } interface BaseCommitResponse { readonly commitSha: string; readonly fixtures: readonly BaseFixture[]; } interface DiffEntry { readonly fixtureId: string; readonly labels?: readonly string[]; } interface ChangedDiffEntry extends DiffEntry { readonly beforeHash: string; readonly afterHash: string; readonly afterPath: string; } interface AddedDiffEntry extends DiffEntry { readonly afterHash: string; readonly afterPath: string; } interface RemovedDiffEntry extends DiffEntry { readonly beforeHash: string; } interface ErroredDiffEntry extends DiffEntry { readonly errorMessage: string; readonly errorStack?: string; } interface DiffResult { readonly changed: readonly ChangedDiffEntry[]; readonly added: readonly AddedDiffEntry[]; readonly removed: readonly RemovedDiffEntry[]; readonly errored: readonly ErroredDiffEntry[]; } function shouldIncludeInReport(labels: readonly string[] | undefined): boolean { return !labels?.some(l => EXCLUDED_LABELS.has(l)); } function collectErrored(local: LocalManifest): ErroredDiffEntry[] { const errored: ErroredDiffEntry[] = []; for (const cur of local.fixtures) { if (!cur.hasError && cur.imageHash && cur.imagePath) { continue; } const errorEvents = (cur.events ?? []).filter(e => e.isError); const errorMessage = errorEvents.map(e => e.message).filter(Boolean).join('; ') || 'unknown error (no image hash produced)'; const errorStack = errorEvents.map(e => e.stack).find(s => s !== undefined); errored.push({ fixtureId: cur.fixtureId, labels: cur.labels, errorMessage, errorStack, }); } return errored; } function diffManifests(local: LocalManifest, base: BaseCommitResponse | null): DiffResult { const errored = collectErrored(local); if (!base) { return { changed: [], added: [], removed: [], errored }; } const baseByFixture = new Map(); for (const f of base.fixtures) { baseByFixture.set(f.fixtureId, f); } const localByFixture = new Map(); for (const f of local.fixtures) { localByFixture.set(f.fixtureId, f); } const changed: ChangedDiffEntry[] = []; const added: AddedDiffEntry[] = []; const removed: RemovedDiffEntry[] = []; for (const cur of local.fixtures) { if (cur.hasError || !cur.imageHash || !cur.imagePath) { continue; } const baseEntry = baseByFixture.get(cur.fixtureId); if (!baseEntry) { added.push({ fixtureId: cur.fixtureId, labels: cur.labels, afterHash: cur.imageHash, afterPath: cur.imagePath, }); continue; } if (baseEntry.imageHash !== cur.imageHash) { changed.push({ fixtureId: cur.fixtureId, labels: cur.labels, beforeHash: baseEntry.imageHash, afterHash: cur.imageHash, afterPath: cur.imagePath, }); } } for (const baseEntry of base.fixtures) { if (!localByFixture.has(baseEntry.fixtureId)) { removed.push({ fixtureId: baseEntry.fixtureId, labels: baseEntry.labels, beforeHash: baseEntry.imageHash, }); } } return { changed, added, removed, errored }; } function loadImageUrl(serviceUrl: string, hash: string): string { return `${serviceUrl.replace(/\/$/, '')}/images/${hash}`; } function generateMarkdown( diff: DiffResult, serviceUrl: string, baseSha: string | null, currentSha: string, pixelDiffs: ReadonlyMap, ): string { const changed = diff.changed.filter(e => shouldIncludeInReport(e.labels)); const added = diff.added.filter(e => shouldIncludeInReport(e.labels)); const removed = diff.removed.filter(e => shouldIncludeInReport(e.labels)); const errored = diff.errored.filter(e => shouldIncludeInReport(e.labels)); const significantChanged = changed.filter(e => { const pd = pixelDiffs.get(e.fixtureId); return !pd || pd.isSignificant; }); const insignificantChangedCount = changed.length - significantChanged.length; if (significantChanged.length === 0 && added.length === 0 && removed.length === 0 && errored.length === 0) { if (insignificantChangedCount > 0) { console.error(`All ${insignificantChangedCount} changed screenshot(s) are insignificant — suppressing PR comment.`); } return ''; } const lines: string[] = []; lines.push('## Screenshot Changes'); lines.push(''); if (baseSha) { lines.push(`**Base:** \`${baseSha.slice(0, 8)}\` **Current:** \`${currentSha.slice(0, 8)}\``); } else { lines.push(`**Current:** \`${currentSha.slice(0, 8)}\` _(base manifest unavailable — only rendering errors are reported)_`); } lines.push(''); if (significantChanged.length > 0 || insignificantChangedCount > 0) { if (significantChanged.length > 0) { lines.push(`### Changed (${significantChanged.length})`); lines.push(''); for (let i = 0; i < significantChanged.length; i++) { const entry = significantChanged[i]; const open = i < EXPAND_FIRST_N ? ' open' : ''; lines.push(`${entry.fixtureId}`); lines.push(''); lines.push('| Before | After |'); lines.push('|--------|-------|'); lines.push(`| ![before](${loadImageUrl(serviceUrl, entry.beforeHash)}) | ![after](${loadImageUrl(serviceUrl, entry.afterHash)}) |`); lines.push(''); lines.push(''); lines.push(''); } } if (insignificantChangedCount > 0) { lines.push(`_${insignificantChangedCount} insignificant change(s) omitted (\u2264${MAX_INSIGNIFICANT_PIXELS} px, \u0394\u2264${MAX_INSIGNIFICANT_CHANNEL_DELTA}). See CI logs for details._`); lines.push(''); } } if (added.length > 0) { lines.push(`### Added (${added.length})`); lines.push(''); for (let i = 0; i < added.length; i++) { const entry = added[i]; const open = i < EXPAND_FIRST_N ? ' open' : ''; lines.push(`${entry.fixtureId}`); lines.push(''); lines.push(`![current](${loadImageUrl(serviceUrl, entry.afterHash)})`); lines.push(''); lines.push(''); lines.push(''); } } if (removed.length > 0) { lines.push(`### Removed (${removed.length})`); lines.push(''); for (let i = 0; i < removed.length; i++) { const entry = removed[i]; const open = i < EXPAND_FIRST_N ? ' open' : ''; lines.push(`${entry.fixtureId}`); lines.push(''); lines.push(`![baseline](${loadImageUrl(serviceUrl, entry.beforeHash)})`); lines.push(''); lines.push(''); lines.push(''); } } if (errored.length > 0) { lines.push(`### Errored (${errored.length})`); lines.push(''); lines.push('Fixtures that failed to render — no screenshot was produced.'); lines.push(''); const reservedForFooter = 200; const usedBytes = () => Buffer.byteLength(lines.join('\n'), 'utf8'); let truncatedAt = -1; for (let i = 0; i < errored.length; i++) { const entry = errored[i]; const open = i < EXPAND_FIRST_N ? ' open' : ''; const headerMessage = entry.errorMessage.split('\n').map(l => l.trim()).find(l => l.length > 0) ?? entry.errorMessage; const header = `${entry.fixtureId} — ${escapeMarkdown(headerMessage)}`; const fullStack = entry.errorStack ?? entry.errorMessage; const fullBlock = `${header}\n\n\`\`\`\n${fullStack}\n\`\`\`\n\n\n`; const remainingBudget = MAX_BODY_BYTES - usedBytes() - reservedForFooter; if (Buffer.byteLength(fullBlock, 'utf8') <= remainingBudget) { lines.push(header); lines.push(''); lines.push('```'); lines.push(fullStack); lines.push('```'); lines.push(''); lines.push(''); lines.push(''); continue; } const minimalBlock = `${header}\n\n\`\`\`\n…\n\`\`\`\n\n\n`; const minimalBudget = remainingBudget - Buffer.byteLength(minimalBlock, 'utf8'); if (minimalBudget < 0) { truncatedAt = i; break; } const truncationMarker = '\n…(truncated)'; const stackBudget = minimalBudget + Buffer.byteLength('…', 'utf8') - Buffer.byteLength(truncationMarker, 'utf8'); const truncatedStack = stackBudget > 0 ? fullStack.slice(0, stackBudget) + truncationMarker : '…(truncated)'; lines.push(header); lines.push(''); lines.push('```'); lines.push(truncatedStack); lines.push('```'); lines.push(''); lines.push(''); lines.push(''); } if (truncatedAt !== -1) { lines.push(`_…and ${errored.length - truncatedAt} more errored fixture(s) omitted to stay under the ${Math.round(MAX_BODY_BYTES / 1024)}KB body limit. See workflow logs for the full list._`); lines.push(''); } } return lines.join('\n'); } function escapeMarkdown(text: string): string { return text.replace(/[\r\n]+/g, ' ').replace(/[<>]/g, c => c === '<' ? '<' : '>'); } async function main(): Promise { const [serviceUrl, baseShaArg, currentSha, baseManifestPathArg, localManifestPath] = process.argv.slice(2); if (!serviceUrl || !currentSha || !localManifestPath) { console.error('Usage: node build/lib/screenshotDiffReport.ts '); process.exit(1); } if (!fs.existsSync(localManifestPath)) { console.error(`Local manifest not found: ${localManifestPath}`); process.exit(1); } const baseManifestPath = baseManifestPathArg && fs.existsSync(baseManifestPathArg) ? baseManifestPathArg : null; if (baseManifestPathArg && !baseManifestPath) { console.error(`Base manifest not found: ${baseManifestPathArg}. Reporting fixture errors only.`); } const baseSha = baseManifestPath ? baseShaArg : null; const local = JSON.parse(fs.readFileSync(localManifestPath, 'utf8')) as LocalManifest; const base = baseManifestPath ? JSON.parse(fs.readFileSync(baseManifestPath, 'utf8')) as BaseCommitResponse : null; const diff = diffManifests(local, base); console.error(`Compare result: ${diff.changed.length} changed, ${diff.added.length} added, ${diff.removed.length} removed, ${diff.errored.length} errored.`); const localManifestDir = path.dirname(path.resolve(localManifestPath)); const pixelDiffs = await computePixelDiffs(diff.changed, serviceUrl, localManifestDir); for (const entry of diff.changed) { const pd = pixelDiffs.get(entry.fixtureId); console.error(` ${entry.fixtureId}: ${pd?.toString() ?? 'pixel diff unavailable'}`); } const tmpDir = path.join(__dirname, '../../.tmp'); fs.mkdirSync(tmpDir, { recursive: true }); fs.writeFileSync( path.join(tmpDir, 'screenshotDiffReport.json'), JSON.stringify(diff, null, 2), ); const markdown = generateMarkdown(diff, serviceUrl, baseSha, currentSha, pixelDiffs); if (!markdown) { console.error('No reportable changes.'); process.exit(0); } process.stdout.write(`${COMMENT_MARKER}\n${markdown}`); } main().catch((err) => { console.error(err); process.exit(1); });