diff --git a/.github/workflows/screenshot-test.yml b/.github/workflows/screenshot-test.yml index c7b5935e2b8..7ee9a703c98 100644 --- a/.github/workflows/screenshot-test.yml +++ b/.github/workflows/screenshot-test.yml @@ -11,6 +11,7 @@ on: permissions: contents: read statuses: write + pull-requests: write concurrency: group: screenshots-${{ github.event.pull_request.number || github.sha }} @@ -72,6 +73,58 @@ jobs: "artifactName": "screenshots" }' + - name: Diff screenshots against merge base + id: diff + if: github.event_name == 'pull_request' + run: | + BODY=$(node build/lib/screenshotDiffReport.ts \ + https://hediet-screenshots.azurewebsites.net \ + ${{ github.repository_owner }} \ + ${{ github.event.repository.name }} \ + ${{ github.event.pull_request.base.sha }} \ + ${{ github.sha }}) + if [ -n "$BODY" ]; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + echo "body<> "$GITHUB_OUTPUT" + echo "$BODY" >> "$GITHUB_OUTPUT" + echo "SCREENSHOT_EOF" >> "$GITHUB_OUTPUT" + fi + continue-on-error: true + + - name: Post PR comment + if: github.event_name == 'pull_request' && steps.diff.outputs.has_changes == 'true' + uses: actions/github-script@v7 + with: + script: | + const marker = ''; + const body = process.env.COMMENT_BODY; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find(c => c.body?.startsWith(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + env: + COMMENT_BODY: ${{ steps.diff.outputs.body }} + # - name: Compare screenshots # id: compare # run: | diff --git a/build/lib/screenshotDiffReport.ts b/build/lib/screenshotDiffReport.ts new file mode 100644 index 00000000000..d79cbf9f922 --- /dev/null +++ b/build/lib/screenshotDiffReport.ts @@ -0,0 +1,155 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Fetches a screenshot diff from the service and prints the PR comment markdown to stdout. +// Usage: node build/lib/screenshotDiffReport.ts +// Outputs nothing (exit 0) when there are no visual changes. + +import * as fs from 'fs'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const COMMENT_MARKER = ''; +const EXPAND_FIRST_N = 5; +const EXCLUDED_LABELS = new Set(['animated', 'flaky']); + +interface CompareEntry { + readonly fixtureId: string; + readonly imageUrl: string; + readonly labels?: readonly string[]; + readonly changeCount?: number; +} + +interface CompareChangedEntry { + readonly fixtureId: string; + readonly beforeImageUrl: string; + readonly afterImageUrl: string; + readonly labels?: readonly string[]; + readonly changeCount?: number; +} + +interface CompareResult { + readonly baseCommitSha: string; + readonly added: readonly CompareEntry[]; + readonly removed: readonly CompareEntry[]; + readonly changed: readonly CompareChangedEntry[]; + readonly unchanged: readonly CompareEntry[]; +} + +function hasExcludedLabel(labels: readonly string[] | undefined): boolean { + return labels?.some(l => EXCLUDED_LABELS.has(l)) ?? false; +} + +function generateMarkdown(result: CompareResult, baseSha: string, currentSha: string): string { + const changed = result.changed.filter(e => !hasExcludedLabel(e.labels)); + const added = result.added.filter(e => !hasExcludedLabel(e.labels)); + const removed = result.removed.filter(e => !hasExcludedLabel(e.labels)); + + if (changed.length === 0 && added.length === 0 && removed.length === 0) { + return ''; + } + + const lines: string[] = []; + + lines.push('## Screenshot Changes'); + lines.push(''); + lines.push(`**Base:** \`${baseSha.slice(0, 8)}\` **Current:** \`${currentSha.slice(0, 8)}\``); + lines.push(''); + + if (changed.length > 0) { + lines.push(`### Changed (${changed.length})`); + lines.push(''); + for (let i = 0; i < changed.length; i++) { + const entry = changed[i]; + const open = i < EXPAND_FIRST_N ? ' open' : ''; + lines.push(`${entry.fixtureId}`); + lines.push(''); + lines.push('| Before | After |'); + lines.push('|--------|-------|'); + lines.push(`| ![before](${entry.beforeImageUrl}) | ![after](${entry.afterImageUrl}) |`); + lines.push(''); + lines.push(''); + lines.push(''); + } + } + + if (added.length > 0) { + lines.push(`### Added (${added.length})`); + lines.push(''); + for (let i = 0; i < added.length; i++) { + const entry = added[i]; + const open = i < EXPAND_FIRST_N ? ' open' : ''; + lines.push(`${entry.fixtureId}`); + lines.push(''); + lines.push(`![current](${entry.imageUrl})`); + lines.push(''); + lines.push(''); + lines.push(''); + } + } + + if (removed.length > 0) { + lines.push(`### Removed (${removed.length})`); + lines.push(''); + for (let i = 0; i < removed.length; i++) { + const entry = removed[i]; + const open = i < EXPAND_FIRST_N ? ' open' : ''; + lines.push(`${entry.fixtureId}`); + lines.push(''); + lines.push(`![baseline](${entry.imageUrl})`); + lines.push(''); + lines.push(''); + lines.push(''); + } + } + + return lines.join('\n'); +} + +async function fetchCompare(serviceUrl: string, owner: string, repo: string, baseSha: string, currentSha: string): Promise { + const response = await fetch(`${serviceUrl}/compare`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ owner, repo, baseCommitSha: baseSha, currentCommitSha: currentSha }), + }); + + if (!response.ok) { + const body = await response.json().catch(() => ({})) as { error?: string }; + throw new Error(body.error ?? `Service returned ${response.status}`); + } + + const result = await response.json() as CompareResult; + + // Write result to .tmp for debugging + const tmpDir = path.join(__dirname, '../../.tmp'); + fs.mkdirSync(tmpDir, { recursive: true }); + fs.writeFileSync(path.join(tmpDir, 'screenshotDiffReport.json'), JSON.stringify(result, null, 2)); + + return result; +} + +async function main(): Promise { + const [serviceUrl, owner, repo, baseSha, currentSha] = process.argv.slice(2); + if (!serviceUrl || !owner || !repo || !baseSha || !currentSha) { + console.error('Usage: node build/lib/screenshotDiffReport.ts '); + process.exit(1); + } + + const result = await fetchCompare(serviceUrl, owner, repo, baseSha, currentSha); + const markdown = generateMarkdown(result, baseSha, currentSha); + + if (!markdown) { + process.exit(0); + } + + process.stdout.write(`${COMMENT_MARKER}\n${markdown}`); +} + +main().catch(err => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +});