jonny
/
Vector-Local
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update reporter to include screenshot, remove try statement to ensure CI fails when an error occurs 

Change-Id: I685c8a74352b4172d7a086467a9b3cfc84d57cee
This commit is contained in:
bwang 2022-05-05 12:35:29 -05:00
parent cdd144f928
commit 6b814da689
2 changed files with 63 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/a11y/reporter/report.mustache
View File

@ -89,9 +89,10 @@
<div class="page">
<h1>Accessibility Report For {{{name}}}</h1>
<p>Run on: {{{pageUrl}}}</p>
<p>Generated at: {{date}}</p>
<h1>Accessibility Report for "{{{name}}}"</h1>
<p><b>Run on:</b> {{{pageUrl}}}</p>
<p><b>Generated at:</b> {{date}}</p>
<p><b>Page screenshot:</b> <a href="./{{{name}}}.png">{{{name}}}.png</a></p>
<nav class="counts">
<a href="#error" class="count error">{{errorCount}} total errors</a>
@ -101,12 +102,12 @@
{{#issueData}}
<div id="{{type}}" class="container">
<h2>{{typeLabel}}, {{typeCount}} rules</h2>
<h2>{{typeLabel}}, {{typeCount}} rule(s)</h2>
{{#messages}}
<div class="message {{type}}">
<h3>{{message}}</h3>
<p>Rule: {{runner}}, {{code}}</p>
{{#runnerExtras}}<p>Impact: {{impact}}</p>{{/runnerExtras}}
<p><b>Rule:</b> {{runner}}, {{code}}</p>
{{#runnerExtras}}<p><b>Impact:</b> {{impact}}</p>{{/runnerExtras}}
<h4>{{issueCount}} instance(s):</h4>
<ul class="issues-list">
{{#issues}}

118
tests/a11y/runA11yTests.js
View File

@ -46,69 +46,63 @@ async function runTests( opts ) {
throw new Error( 'Missing env variables' );
}
try {
const tests = config.tests;
const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
if ( !allValidTests ) {
throw new Error( 'Config missing test name' );
}
const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
if ( opts.logResults && !canLogResults ) {
throw new Error( 'Unable to log results, missing config or env variables' );
}
resetReportDir();
const testPromises = tests.map( ( test ) => {
const { url, name, ...testOptions } = test;
const options = { ...config.defaults, ...testOptions };
// Automatically enable screen capture for every test;
options.screenCapture = `${config.reportDir}/${name}.png`;
return pa11y( url, options ).then( ( testResult ) => {
testResult.name = name;
return testResult;
} );
} );
// Run tests against multiple URLs
const results = await Promise.all( testPromises ); // eslint-disable-line
results.forEach( async ( testResult ) => {
const name = testResult.name;
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
// Log results summary to console
if ( !opts.silent ) {
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
}
// Send data to Graphite
// WMF_JENKINS_BEACON_URL is only defined in CI env
if ( opts.logResults && canLogResults ) {
await sendMetrics( config.namespace, testResult.name, errorNum )
.then( ( response ) => {
if ( response.ok ) {
console.log( `'${name}' results logged successfully` );
} else {
console.error( `Failed to log '${name}' results` );
}
} );
}
// Save in html report
const html = await htmlReporter.results( testResult );
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
// Save in json report
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
} );
} catch ( error ) {
// Output an error if it occurred
console.error( error.message );
const tests = config.tests;
const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
if ( !allValidTests ) {
throw new Error( 'Config missing test name' );
}
const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
if ( opts.logResults && !canLogResults ) {
throw new Error( 'Unable to log results, missing config or env variables' );
}
resetReportDir();
const testPromises = tests.map( ( test ) => {
const { url, name, ...testOptions } = test;
const options = { ...config.defaults, ...testOptions };
// Automatically enable screen capture for every test;
options.screenCapture = `${config.reportDir}/${name}.png`;
return pa11y( url, options ).then( ( testResult ) => {
testResult.name = name;
return testResult;
} );
} );
// Run tests against multiple URLs
const results = await Promise.all( testPromises ); // eslint-disable-line
results.forEach( async ( testResult ) => {
const name = testResult.name;
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
// Log results summary to console
if ( !opts.silent ) {
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
}
// Send data to Graphite
// WMF_JENKINS_BEACON_URL is only defined in CI env
if ( opts.logResults && canLogResults ) {
await sendMetrics( config.namespace, testResult.name, errorNum )
.then( ( response ) => {
if ( response.ok ) {
console.log( `'${name}' results logged successfully` );
} else {
console.error( `Failed to log '${name}' results` );
}
} );
}
// Save in html report
const html = await htmlReporter.results( testResult );
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
// Save in json report
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
} );
}
function setupCLI() {