Update reporter to include screenshot, remove try statement to ensure CI fails when an error occurs
Change-Id: I685c8a74352b4172d7a086467a9b3cfc84d57cee
This commit is contained in:
parent
cdd144f928
commit
6b814da689
|
@ -89,9 +89,10 @@
|
||||||
|
|
||||||
<div class="page">
|
<div class="page">
|
||||||
|
|
||||||
<h1>Accessibility Report For {{{name}}}</h1>
|
<h1>Accessibility Report for "{{{name}}}"</h1>
|
||||||
<p>Run on: {{{pageUrl}}}</p>
|
<p><b>Run on:</b> {{{pageUrl}}}</p>
|
||||||
<p>Generated at: {{date}}</p>
|
<p><b>Generated at:</b> {{date}}</p>
|
||||||
|
<p><b>Page screenshot:</b> <a href="./{{{name}}}.png">{{{name}}}.png</a></p>
|
||||||
|
|
||||||
<nav class="counts">
|
<nav class="counts">
|
||||||
<a href="#error" class="count error">{{errorCount}} total errors</a>
|
<a href="#error" class="count error">{{errorCount}} total errors</a>
|
||||||
|
@ -101,12 +102,12 @@
|
||||||
|
|
||||||
{{#issueData}}
|
{{#issueData}}
|
||||||
<div id="{{type}}" class="container">
|
<div id="{{type}}" class="container">
|
||||||
<h2>{{typeLabel}}, {{typeCount}} rules</h2>
|
<h2>{{typeLabel}}, {{typeCount}} rule(s)</h2>
|
||||||
{{#messages}}
|
{{#messages}}
|
||||||
<div class="message {{type}}">
|
<div class="message {{type}}">
|
||||||
<h3>{{message}}</h3>
|
<h3>{{message}}</h3>
|
||||||
<p>Rule: {{runner}}, {{code}}</p>
|
<p><b>Rule:</b> {{runner}}, {{code}}</p>
|
||||||
{{#runnerExtras}}<p>Impact: {{impact}}</p>{{/runnerExtras}}
|
{{#runnerExtras}}<p><b>Impact:</b> {{impact}}</p>{{/runnerExtras}}
|
||||||
<h4>{{issueCount}} instance(s):</h4>
|
<h4>{{issueCount}} instance(s):</h4>
|
||||||
<ul class="issues-list">
|
<ul class="issues-list">
|
||||||
{{#issues}}
|
{{#issues}}
|
||||||
|
|
|
@ -46,69 +46,63 @@ async function runTests( opts ) {
|
||||||
throw new Error( 'Missing env variables' );
|
throw new Error( 'Missing env variables' );
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
const tests = config.tests;
|
||||||
const tests = config.tests;
|
const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
|
||||||
const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
|
if ( !allValidTests ) {
|
||||||
if ( !allValidTests ) {
|
throw new Error( 'Config missing test name' );
|
||||||
throw new Error( 'Config missing test name' );
|
|
||||||
}
|
|
||||||
|
|
||||||
const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
|
|
||||||
if ( opts.logResults && !canLogResults ) {
|
|
||||||
throw new Error( 'Unable to log results, missing config or env variables' );
|
|
||||||
}
|
|
||||||
|
|
||||||
resetReportDir();
|
|
||||||
|
|
||||||
const testPromises = tests.map( ( test ) => {
|
|
||||||
const { url, name, ...testOptions } = test;
|
|
||||||
const options = { ...config.defaults, ...testOptions };
|
|
||||||
// Automatically enable screen capture for every test;
|
|
||||||
options.screenCapture = `${config.reportDir}/${name}.png`;
|
|
||||||
|
|
||||||
return pa11y( url, options ).then( ( testResult ) => {
|
|
||||||
testResult.name = name;
|
|
||||||
return testResult;
|
|
||||||
} );
|
|
||||||
} );
|
|
||||||
|
|
||||||
// Run tests against multiple URLs
|
|
||||||
const results = await Promise.all( testPromises ); // eslint-disable-line
|
|
||||||
results.forEach( async ( testResult ) => {
|
|
||||||
const name = testResult.name;
|
|
||||||
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
|
|
||||||
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
|
|
||||||
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
|
|
||||||
|
|
||||||
// Log results summary to console
|
|
||||||
if ( !opts.silent ) {
|
|
||||||
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send data to Graphite
|
|
||||||
// WMF_JENKINS_BEACON_URL is only defined in CI env
|
|
||||||
if ( opts.logResults && canLogResults ) {
|
|
||||||
await sendMetrics( config.namespace, testResult.name, errorNum )
|
|
||||||
.then( ( response ) => {
|
|
||||||
if ( response.ok ) {
|
|
||||||
console.log( `'${name}' results logged successfully` );
|
|
||||||
} else {
|
|
||||||
console.error( `Failed to log '${name}' results` );
|
|
||||||
}
|
|
||||||
} );
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save in html report
|
|
||||||
const html = await htmlReporter.results( testResult );
|
|
||||||
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
|
|
||||||
// Save in json report
|
|
||||||
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
|
|
||||||
} );
|
|
||||||
|
|
||||||
} catch ( error ) {
|
|
||||||
// Output an error if it occurred
|
|
||||||
console.error( error.message );
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
|
||||||
|
if ( opts.logResults && !canLogResults ) {
|
||||||
|
throw new Error( 'Unable to log results, missing config or env variables' );
|
||||||
|
}
|
||||||
|
|
||||||
|
resetReportDir();
|
||||||
|
|
||||||
|
const testPromises = tests.map( ( test ) => {
|
||||||
|
const { url, name, ...testOptions } = test;
|
||||||
|
const options = { ...config.defaults, ...testOptions };
|
||||||
|
// Automatically enable screen capture for every test;
|
||||||
|
options.screenCapture = `${config.reportDir}/${name}.png`;
|
||||||
|
|
||||||
|
return pa11y( url, options ).then( ( testResult ) => {
|
||||||
|
testResult.name = name;
|
||||||
|
return testResult;
|
||||||
|
} );
|
||||||
|
} );
|
||||||
|
|
||||||
|
// Run tests against multiple URLs
|
||||||
|
const results = await Promise.all( testPromises ); // eslint-disable-line
|
||||||
|
results.forEach( async ( testResult ) => {
|
||||||
|
const name = testResult.name;
|
||||||
|
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
|
||||||
|
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
|
||||||
|
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
|
||||||
|
|
||||||
|
// Log results summary to console
|
||||||
|
if ( !opts.silent ) {
|
||||||
|
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send data to Graphite
|
||||||
|
// WMF_JENKINS_BEACON_URL is only defined in CI env
|
||||||
|
if ( opts.logResults && canLogResults ) {
|
||||||
|
await sendMetrics( config.namespace, testResult.name, errorNum )
|
||||||
|
.then( ( response ) => {
|
||||||
|
if ( response.ok ) {
|
||||||
|
console.log( `'${name}' results logged successfully` );
|
||||||
|
} else {
|
||||||
|
console.error( `Failed to log '${name}' results` );
|
||||||
|
}
|
||||||
|
} );
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save in html report
|
||||||
|
const html = await htmlReporter.results( testResult );
|
||||||
|
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
|
||||||
|
// Save in json report
|
||||||
|
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
|
||||||
|
} );
|
||||||
}
|
}
|
||||||
|
|
||||||
function setupCLI() {
|
function setupCLI() {
|
||||||
|
|
Loading…
Reference in New Issue