From 6b814da689328137a925047ec944d521f521f1c6 Mon Sep 17 00:00:00 2001
From: bwang <bwang@wikimedia.org>
Date: Thu, 5 May 2022 12:35:29 -0500
Subject: [PATCH] Update reporter to include screenshot, remove try statement
 to ensure CI fails when an error occurs

Change-Id: I685c8a74352b4172d7a086467a9b3cfc84d57cee
---
 tests/a11y/reporter/report.mustache |  13 +--
 tests/a11y/runA11yTests.js          | 118 +++++++++++++---------------
 2 files changed, 63 insertions(+), 68 deletions(-)

diff --git a/tests/a11y/reporter/report.mustache b/tests/a11y/reporter/report.mustache
index 607d0ddc..93bd07a5 100644
--- a/tests/a11y/reporter/report.mustache
+++ b/tests/a11y/reporter/report.mustache
@@ -89,9 +89,10 @@
 
 	<div class="page">
 
-		<h1>Accessibility Report For {{{name}}}</h1>
-		<p>Run on: {{{pageUrl}}}</p>
-		<p>Generated at: {{date}}</p>
+		<h1>Accessibility Report for "{{{name}}}"</h1>
+		<p><b>Run on:</b> {{{pageUrl}}}</p>
+		<p><b>Generated at:</b> {{date}}</p>
+		<p><b>Page screenshot:</b> <a href="./{{{name}}}.png">{{{name}}}.png</a></p>
 
 		<nav class="counts">
 			<a href="#error" class="count error">{{errorCount}} total errors</a>
@@ -101,12 +102,12 @@
 
 		{{#issueData}}
 		<div id="{{type}}" class="container">
-			<h2>{{typeLabel}}, {{typeCount}} rules</h2>
+			<h2>{{typeLabel}}, {{typeCount}} rule(s)</h2>
 			{{#messages}}
 			<div class="message {{type}}">
 				<h3>{{message}}</h3>
-				<p>Rule: {{runner}}, {{code}}</p>
-				{{#runnerExtras}}<p>Impact: {{impact}}</p>{{/runnerExtras}}
+				<p><b>Rule:</b> {{runner}}, {{code}}</p>
+				{{#runnerExtras}}<p><b>Impact:</b> {{impact}}</p>{{/runnerExtras}}
 				<h4>{{issueCount}} instance(s):</h4>
 				<ul class="issues-list">
 					{{#issues}}
diff --git a/tests/a11y/runA11yTests.js b/tests/a11y/runA11yTests.js
index 42995290..bb586c19 100644
--- a/tests/a11y/runA11yTests.js
+++ b/tests/a11y/runA11yTests.js
@@ -46,69 +46,63 @@ async function runTests( opts ) {
 		throw new Error( 'Missing env variables' );
 	}
 
-	try {
-		const tests = config.tests;
-		const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
-		if ( !allValidTests ) {
-			throw new Error( 'Config missing test name' );
-		}
-
-		const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
-		if ( opts.logResults && !canLogResults ) {
-			throw new Error( 'Unable to log results, missing config or env variables' );
-		}
-
-		resetReportDir();
-
-		const testPromises = tests.map( ( test ) => {
-			const { url, name, ...testOptions } = test;
-			const options = { ...config.defaults, ...testOptions };
-			// Automatically enable screen capture for every test;
-			options.screenCapture = `${config.reportDir}/${name}.png`;
-
-			return pa11y( url, options ).then( ( testResult ) => {
-				testResult.name = name;
-				return testResult;
-			} );
-		} );
-
-		// Run tests against multiple URLs
-		const results = await Promise.all( testPromises ); // eslint-disable-line
-		results.forEach( async ( testResult ) => {
-			const name = testResult.name;
-			const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
-			const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
-			const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
-
-			// Log results summary to console
-			if ( !opts.silent ) {
-				console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
-			}
-
-			// Send data to Graphite
-			// WMF_JENKINS_BEACON_URL is only defined in CI env
-			if ( opts.logResults && canLogResults ) {
-				await sendMetrics( config.namespace, testResult.name, errorNum )
-					.then( ( response ) => {
-						if ( response.ok ) {
-							console.log( `'${name}' results logged successfully` );
-						} else {
-							console.error( `Failed to log '${name}' results` );
-						}
-					} );
-			}
-
-			// Save in html report
-			const html = await htmlReporter.results( testResult );
-			fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
-			// Save in json report
-			fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, '  ' ), 'utf8' );
-		} );
-
-	} catch ( error ) {
-		// Output an error if it occurred
-		console.error( error.message );
+	const tests = config.tests;
+	const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
+	if ( !allValidTests ) {
+		throw new Error( 'Config missing test name' );
 	}
+
+	const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
+	if ( opts.logResults && !canLogResults ) {
+		throw new Error( 'Unable to log results, missing config or env variables' );
+	}
+
+	resetReportDir();
+
+	const testPromises = tests.map( ( test ) => {
+		const { url, name, ...testOptions } = test;
+		const options = { ...config.defaults, ...testOptions };
+		// Automatically enable screen capture for every test;
+		options.screenCapture = `${config.reportDir}/${name}.png`;
+
+		return pa11y( url, options ).then( ( testResult ) => {
+			testResult.name = name;
+			return testResult;
+		} );
+	} );
+
+	// Run tests against multiple URLs
+	const results = await Promise.all( testPromises ); // eslint-disable-line
+	results.forEach( async ( testResult ) => {
+		const name = testResult.name;
+		const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
+		const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
+		const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
+
+		// Log results summary to console
+		if ( !opts.silent ) {
+			console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
+		}
+
+		// Send data to Graphite
+		// WMF_JENKINS_BEACON_URL is only defined in CI env
+		if ( opts.logResults && canLogResults ) {
+			await sendMetrics( config.namespace, testResult.name, errorNum )
+				.then( ( response ) => {
+					if ( response.ok ) {
+						console.log( `'${name}' results logged successfully` );
+					} else {
+						console.error( `Failed to log '${name}' results` );
+					}
+				} );
+		}
+
+		// Save in html report
+		const html = await htmlReporter.results( testResult );
+		fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
+		// Save in json report
+		fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, '  ' ), 'utf8' );
+	} );
 }
 
 function setupCLI() {