Add automated a11y tests with pa11y
Bug: T301184 Change-Id: If3d5aa74b9b1f5bf0f7aa2ea6950853c9a5f5ada
This commit is contained in:
bwang
Jdlrobson
parent
094928736c
commit
dd9b6e1dbc
File diff suppressed because it is too large
Load Diff
|
|
@ -4,13 +4,15 @@
|
|||
"start": "bash dev-scripts/setup-storybook.sh && start-storybook --quiet -p 6006 -s resources/skins.vector.styles",
|
||||
"test": "npm -s run lint && tsc && npm run test:unit && npm -s run doc",
|
||||
"test:unit": "jest --silent",
|
||||
"test:a11y": "node tests/a11y/runA11yTests.js --env development",
|
||||
"selenium-daily": "node tests/a11y/runA11yTests.js --env ci --logResults",
|
||||
"lint": "npm -s run lint:js && npm -s run lint:styles && npm -s run lint:i18n",
|
||||
"lint:fix:js": "npm -s run lint:js -- --fix",
|
||||
"lint:fix:styles": "npm -s run lint:styles -- --fix",
|
||||
"lint:js": "eslint --cache .",
|
||||
"lint:styles": "stylelint \"**/*.{less,css}\"",
|
||||
"lint:i18n": "banana-checker --requireLowerCase=0 i18n/",
|
||||
"doc": "jsdoc -c jsdoc.json && npm run build-storybook -s resources/skins.vector.styles",
|
||||
"doc": "jsdoc -c jsdoc.json && npm run build-storybook -s resources/skins.vector.styles && npm run test:a11y",
|
||||
"build-storybook": "bash dev-scripts/setup-storybook.sh && build-storybook --quiet --loglevel warn -o docs/ui",
|
||||
"minify-svg": "svgo --config=.svgo.config.js --quiet --recursive --folder resources/",
|
||||
"pre-commit": "[ \"${PRE_COMMIT:-1}\" -eq 0 ] || npm -s t"
|
||||
|
|
@ -28,6 +30,7 @@
|
|||
"@wikimedia/types-wikimedia": "0.2.0",
|
||||
"@wikimedia/wvui": "0.3.5",
|
||||
"babel-loader": "8.0.6",
|
||||
"commander": "9.1.0",
|
||||
"eslint-config-wikimedia": "0.20.0",
|
||||
"grunt-banana-checker": "0.9.0",
|
||||
"jest": "26.4.2",
|
||||
|
|
@ -38,6 +41,7 @@
|
|||
"less-loader": "4.1.0",
|
||||
"mustache": "3.0.1",
|
||||
"node-fetch": "2.6.1",
|
||||
"pa11y": "6.1.1",
|
||||
"pre-commit": "1.2.2",
|
||||
"stylelint-config-wikimedia": "0.11.1",
|
||||
"svgo": "2.8.0",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"extends": [
|
||||
"../../.eslintrcEs6.json"
|
||||
],
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2018
|
||||
},
|
||||
"env": {
|
||||
"node": true
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
// @ts-nocheck
|
||||
const config = {
|
||||
reportDir: 'docs/a11y',
|
||||
namespace: 'Vector',
|
||||
env: {
|
||||
development: {
|
||||
baseUrl: process.env.MW_SERVER,
|
||||
defaultPage: '/wiki/Polar_bear?useskin=vector-2022'
|
||||
},
|
||||
ci: {
|
||||
baseUrl: 'https://en.wikipedia.beta.wmflabs.org',
|
||||
defaultPage: '/wiki/Polar_bear'
|
||||
}
|
||||
},
|
||||
defaults: {
|
||||
viewport: {
|
||||
width: 1200,
|
||||
height: 1080
|
||||
},
|
||||
runners: [
|
||||
'axe',
|
||||
'htmlcs'
|
||||
],
|
||||
includeWarnings: true,
|
||||
includeNotices: true,
|
||||
hideElements: '#content'
|
||||
}
|
||||
};
|
||||
|
||||
config.tests = ( envName ) => ( [
|
||||
{
|
||||
name: 'default',
|
||||
url: config.env[ envName ].baseUrl + config.env[ envName ].defaultPage
|
||||
},
|
||||
{
|
||||
name: 'logged_in',
|
||||
url: config.env[ envName ].baseUrl + config.env[ envName ].defaultPage,
|
||||
wait: '500',
|
||||
actions: [
|
||||
'click #p-personal-checkbox',
|
||||
'wait for .vector-user-menu-login a to be visible',
|
||||
'click .vector-user-menu-login a',
|
||||
'wait for #wpName1 to be visible',
|
||||
'set field #wpName1 to ' + process.env.MEDIAWIKI_USER,
|
||||
'set field #wpPassword1 to ' + process.env.MEDIAWIKI_PASSWORD,
|
||||
'click #wpLoginAttempt',
|
||||
'wait for #pt-userpage-2 to be visible' // Confirm login was successful
|
||||
]
|
||||
},
|
||||
{
|
||||
name: 'search',
|
||||
url: config.env[ envName ].baseUrl + config.env[ envName ].defaultPage,
|
||||
rootElement: '#p-search',
|
||||
wait: '500',
|
||||
actions: [
|
||||
'click #searchInput',
|
||||
'wait for .wvui-input__input to be added',
|
||||
'set field .wvui-input__input to Test'
|
||||
]
|
||||
}
|
||||
] );
|
||||
|
||||
module.exports = config;
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8"/>
|
||||
<title>Accessibility Report For "{{pageUrl}}" ({{date}})</title>
|
||||
|
||||
<style>
|
||||
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background-color: #fff;
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
font-size: 16px;
|
||||
line-height: 22px;
|
||||
color: #333;
|
||||
}
|
||||
li {
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
pre {
|
||||
white-space: pre-wrap;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.page {
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 25px;
|
||||
}
|
||||
|
||||
.counts {
|
||||
margin-top: 30px;
|
||||
font-size: 20px;
|
||||
}
|
||||
.count {
|
||||
display: inline-block;
|
||||
padding: 5px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #eee;
|
||||
}
|
||||
|
||||
.container {
|
||||
margin-top: 30px;
|
||||
}
|
||||
|
||||
.message {
|
||||
margin-top: 20px;
|
||||
padding: 0 15px 0 15px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #eee;
|
||||
}
|
||||
|
||||
.issues-list {
|
||||
margin-left: 0;
|
||||
padding-left: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.issue {
|
||||
border-radius: 5px;
|
||||
padding: 10px;
|
||||
border: 1px solid #eee;
|
||||
background-color: white;
|
||||
}
|
||||
|
||||
pre {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.error {
|
||||
background-color: #fdd;
|
||||
border-color: #ff9696;
|
||||
}
|
||||
.warning {
|
||||
background-color: #ffd;
|
||||
border-color: #e7c12b;
|
||||
}
|
||||
.notice {
|
||||
background-color: #eef4ff;
|
||||
border-color: #b6d0ff;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="page">
|
||||
|
||||
<h1>Accessibility Report For {{{name}}}</h1>
|
||||
<p>Run on: {{{pageUrl}}}</p>
|
||||
<p>Generated at: {{date}}</p>
|
||||
|
||||
<nav class="counts">
|
||||
<a href="#error" class="count error">{{errorCount}} total errors</a>
|
||||
<a href="#warning" class="count warning">{{warningCount}} total warnings</a>
|
||||
<a href="#notice" class="count notice">{{noticeCount}} total notices</a>
|
||||
</nav>
|
||||
|
||||
{{#issueData}}
|
||||
<div id="{{type}}" class="container">
|
||||
<h2>{{typeLabel}}, {{typeCount}} rules</h2>
|
||||
{{#messages}}
|
||||
<div class="message {{type}}">
|
||||
<h3>{{message}}</h3>
|
||||
<p>Rule: {{runner}}, {{code}}</p>
|
||||
{{#runnerExtras}}<p>Impact: {{impact}}</p>{{/runnerExtras}}
|
||||
<h4>{{issueCount}} instance(s):</h4>
|
||||
<ul class="issues-list">
|
||||
{{#issues}}
|
||||
<li class="issue">
|
||||
<pre>{{selector}}</pre>
|
||||
<hr>
|
||||
<pre>{{context}}</pre>
|
||||
</li>
|
||||
{{/issues}}
|
||||
</ul>
|
||||
</div>
|
||||
{{/messages}}
|
||||
</div>
|
||||
{{/issueData}}
|
||||
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
// @ts-nocheck
|
||||
'use strict';
|
||||
|
||||
const fs = require( 'fs' );
|
||||
const path = require( 'path' );
|
||||
const mustache = require( 'mustache' );
|
||||
const reportTemplate = fs.readFileSync( path.resolve( __dirname, 'report.mustache' ), 'utf8' );
|
||||
|
||||
const report = module.exports = {};
|
||||
|
||||
// Utility function to uppercase the first character of a string
|
||||
function upperCaseFirst( string ) {
|
||||
return string.charAt( 0 ).toUpperCase() + string.slice( 1 );
|
||||
}
|
||||
|
||||
// Pa11y version support
|
||||
report.supports = '^6.0.0 || ^6.0.0-alpha || ^6.0.0-beta';
|
||||
|
||||
// Compile template and output formatted results
|
||||
report.results = async ( results ) => {
|
||||
const messagesByType = results.issues.reduce( ( result, issue ) => {
|
||||
if ( result[ issue.type ].indexOf( issue.message ) === -1 ) {
|
||||
result[ issue.type ].push( issue.message );
|
||||
}
|
||||
return result;
|
||||
}, { error: [], warning: [], notice: [] } );
|
||||
const issuesByMessage = results.issues.reduce( ( result, issue ) => {
|
||||
if ( result[ issue.message ] ) {
|
||||
result[ issue.message ].push( issue );
|
||||
} else {
|
||||
result[ issue.message ] = [ issue ];
|
||||
}
|
||||
return result;
|
||||
}, {} );
|
||||
const issueData = [ 'error', 'warning', 'notice' ].map( ( type ) => ( {
|
||||
type,
|
||||
typeLabel: upperCaseFirst( type ) + 's',
|
||||
typeCount: messagesByType[ type ].length,
|
||||
messages: messagesByType[ type ].map( ( message ) => {
|
||||
const firstIssue = issuesByMessage[ message ][ 0 ];
|
||||
const hasRunnerExtras = Object.keys( firstIssue.runnerExtras ).length > 0;
|
||||
return {
|
||||
message,
|
||||
issueCount: issuesByMessage[ message ].length,
|
||||
runner: firstIssue.runner,
|
||||
runnerExtras: hasRunnerExtras ? firstIssue.runnerExtras : false,
|
||||
code: firstIssue.code,
|
||||
issues: issuesByMessage[ message ]
|
||||
};
|
||||
} ).sort( ( a, b ) => {
|
||||
// Sort messages by number of issues
|
||||
return b.issueCount - a.issueCount;
|
||||
} )
|
||||
} ) );
|
||||
|
||||
return mustache.render( reportTemplate, {
|
||||
// The current date
|
||||
date: new Date(),
|
||||
|
||||
// Test information
|
||||
name: results.name,
|
||||
pageUrl: results.pageUrl,
|
||||
|
||||
// Results
|
||||
issueData,
|
||||
|
||||
// Issue counts
|
||||
errorCount: results.issues.filter( ( issue ) => issue.type === 'error' ).length,
|
||||
warningCount: results.issues.filter( ( issue ) => issue.type === 'warning' ).length,
|
||||
noticeCount: results.issues.filter( ( issue ) => issue.type === 'notice' ).length
|
||||
} );
|
||||
};
|
||||
|
||||
// Output error messages
|
||||
report.error = ( message ) => {
|
||||
return message;
|
||||
};
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
/* eslint-disable no-console */
|
||||
// @ts-nocheck
|
||||
const fs = require( 'fs' );
|
||||
const fetch = require( 'node-fetch' );
|
||||
const path = require( 'path' );
|
||||
const pa11y = require( 'pa11y' );
|
||||
|
||||
const htmlReporter = require( path.resolve( __dirname, './reporter/reporter.js' ) );
|
||||
const config = require( path.resolve( __dirname, 'a11y.config.js' ) );
|
||||
|
||||
/**
|
||||
* Delete and recreate the report directory
|
||||
*/
|
||||
function resetReportDir() {
|
||||
// Delete and create report directory
|
||||
fs.rmdirSync( config.reportDir, { recursive: true } );
|
||||
fs.mkdirSync( config.reportDir, { recursive: true } );
|
||||
}
|
||||
|
||||
/**
|
||||
* Log test results to Graphite
|
||||
*
|
||||
* @param {string} namespace
|
||||
* @param {string} name
|
||||
* @param {number} count
|
||||
* @return {Promise<any>}
|
||||
*/
|
||||
function sendMetrics( namespace, name, count ) {
|
||||
const metricPrefix = 'MediaWiki.a11y';
|
||||
const url = `${process.env.BEACON_URL}${metricPrefix}.${namespace}.${name}=${count}c`;
|
||||
return fetch( url );
|
||||
}
|
||||
|
||||
/**
|
||||
* Run pa11y on tests specified by the config.
|
||||
*
|
||||
* @param {Object} opts
|
||||
*/
|
||||
async function runTests( opts ) {
|
||||
try {
|
||||
if ( !config.env[ opts.env ] ) {
|
||||
throw new Error( `Invalid env value: '${opts.env}'` );
|
||||
}
|
||||
|
||||
if ( opts.env !== 'ci' && opts.logResults ) {
|
||||
throw new Error( "Results can only be logged with '--env ci'" );
|
||||
}
|
||||
|
||||
const tests = config.tests( opts.env );
|
||||
const allTestsHaveNames = tests.filter( ( test ) => test.name ).length === tests.length;
|
||||
if ( !allTestsHaveNames ) {
|
||||
throw new Error( 'Config missing test name' );
|
||||
}
|
||||
|
||||
resetReportDir();
|
||||
|
||||
const testPromises = tests.map( ( test ) => {
|
||||
const { url, name, ...testOptions } = test;
|
||||
const options = { ...config.defaults, ...testOptions };
|
||||
// Automatically enable screen capture for every test;
|
||||
options.screenCapture = `${config.reportDir}/${name}.png`;
|
||||
|
||||
return pa11y( url, options ).then( ( testResult ) => {
|
||||
testResult.name = name;
|
||||
return testResult;
|
||||
} );
|
||||
} );
|
||||
|
||||
// Run tests against multiple URLs
|
||||
const results = await Promise.all( testPromises ); // eslint-disable-line
|
||||
results.forEach( async ( testResult ) => {
|
||||
const name = testResult.name;
|
||||
const errorNum = testResult.issues.filter( ( issue ) => issue.type === 'error' ).length;
|
||||
const warningNum = testResult.issues.filter( ( issue ) => issue.type === 'warning' ).length;
|
||||
const noticeNum = testResult.issues.filter( ( issue ) => issue.type === 'notice' ).length;
|
||||
|
||||
// Log results summary to console
|
||||
if ( !opts.silent ) {
|
||||
console.log( `'${name}'- ${errorNum} errors, ${warningNum} warnings, ${noticeNum} notices` );
|
||||
}
|
||||
|
||||
// Send data to Graphite
|
||||
// BEACON_URL is only defined in CI env
|
||||
if ( opts.env === 'ci' && opts.logResults && process.env.BEACON_URL ) {
|
||||
if ( !config.namespace ) {
|
||||
throw new Error( 'Config missing namespace' );
|
||||
}
|
||||
await sendMetrics( config.namespace, testResult.name, errorNum )
|
||||
.then( ( response ) => {
|
||||
if ( response.ok ) {
|
||||
console.log( `'${name}' results logged successfully` );
|
||||
} else {
|
||||
console.error( `Failed to log '${name}' results` );
|
||||
}
|
||||
} );
|
||||
}
|
||||
|
||||
// Save in html report
|
||||
const html = await htmlReporter.results( testResult );
|
||||
fs.promises.writeFile( `${config.reportDir}/report-${name}.html`, html, 'utf8' );
|
||||
// Save in json report
|
||||
fs.promises.writeFile( `${config.reportDir}/report-${name}.json`, JSON.stringify( testResult, null, ' ' ), 'utf8' );
|
||||
} );
|
||||
|
||||
} catch ( error ) {
|
||||
// Output an error if it occurred
|
||||
console.error( error.message );
|
||||
}
|
||||
}
|
||||
|
||||
function setupCLI() {
|
||||
const { program } = require( 'commander' );
|
||||
|
||||
program
|
||||
.requiredOption( '-e, --env <env>', 'determine which urls tests are run on, development or ci' )
|
||||
.option( '-s, --silent', 'avoids logging results summary to console', false )
|
||||
.option( '-l, --logResults', 'log a11y results to Graphite, should only be used with --env ci', false )
|
||||
.action( ( opts ) => {
|
||||
runTests( opts );
|
||||
} );
|
||||
|
||||
program.parse();
|
||||
}
|
||||
|
||||
setupCLI();
|
||||
Loading…
Reference in New Issue