From a77fed0a558f647ea6427c874692366c1b8f20f6 Mon Sep 17 00:00:00 2001
From: bwang <bwang@wikimedia.org>
Date: Fri, 15 Apr 2022 12:03:21 -0500
Subject: [PATCH] Update a11y tests to output files in log/

This makes built assets available in jenkins,
Also addresses miscellaneous feedback relating to configuration
variable name and track topic name.

Change-Id: Ic70d517eba96aab6512a96ea19d6be56b3664871
---
 .eslintignore              |  1 +
 .gitignore                 |  1 +
 package.json               |  2 +-
 tests/a11y/a11y.config.js  | 14 +++++++++-----
 tests/a11y/runA11yTests.js | 16 +++++++++++-----
 5 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/.eslintignore b/.eslintignore
index a81a7ab5..0c41319b 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,5 +1,6 @@
 /.storybook/
 /docs/
+/log/
 /i18n/
 /node_modules/
 /vendor/
diff --git a/.gitignore b/.gitignore
index 71183f81..e4f87eeb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ sftp-config.json
 # Building & testing
 /composer.lock
 /docs
+/log
 /node_modules
 /vendor
 /coverage
diff --git a/package.json b/package.json
index d905b164..c18bccd7 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,7 @@
 		"lint:js": "eslint --cache .",
 		"lint:styles": "stylelint \"**/*.{less,css}\"",
 		"lint:i18n": "banana-checker --requireLowerCase=0 i18n/",
-		"doc": "jsdoc -c jsdoc.json && npm run build-storybook -s resources/skins.vector.styles && npm run test:a11y",
+		"doc": "jsdoc -c jsdoc.json && npm run build-storybook -s resources/skins.vector.styles",
 		"build-storybook": "bash dev-scripts/setup-storybook.sh && build-storybook --quiet --loglevel warn -o docs/ui",
 		"minify-svg": "svgo --config=.svgo.config.js --quiet --recursive --folder resources/",
 		"pre-commit": "[ \"${PRE_COMMIT:-1}\" -eq 0 ] || npm -s t"
diff --git a/tests/a11y/a11y.config.js b/tests/a11y/a11y.config.js
index ab39ac81..5f7de426 100644
--- a/tests/a11y/a11y.config.js
+++ b/tests/a11y/a11y.config.js
@@ -8,7 +8,7 @@ const testData = {
 };
 
 module.exports = {
-	reportDir: 'docs/a11y',
+	reportDir: 'log/a11y',
 	namespace: 'Vector',
 	defaults: {
 		viewport: {
@@ -33,11 +33,14 @@ module.exports = {
 	tests: [
 		{
 			name: 'default',
-			url: testData.baseUrl + testData.defaultPage
+			url: testData.baseUrl + testData.pageUrl,
+			actions: [
+				'click #mw-sidebar-button'
+			]
 		},
 		{
 			name: 'logged_in',
-			url: testData.baseUrl + testData.defaultPage,
+			url: testData.baseUrl + testData.pageUrl,
 			wait: '500',
 			actions: [
 				'click #p-personal-checkbox',
@@ -47,12 +50,13 @@ module.exports = {
 				'set field #wpName1 to ' + testData.loginUser,
 				'set field #wpPassword1 to ' + testData.loginPassword,
 				'click #wpLoginAttempt',
-				'wait for #pt-userpage-2 to be visible' // Confirm login was successful
+				'wait for #pt-userpage-2 to be visible', // Confirm login was successful
+				'click #mw-sidebar-button'
 			]
 		},
 		{
 			name: 'search',
-			url: testData.baseUrl + testData.defaultPage,
+			url: testData.baseUrl + testData.pageUrl,
 			rootElement: '#p-search',
 			wait: '500',
 			actions: [
diff --git a/tests/a11y/runA11yTests.js b/tests/a11y/runA11yTests.js
index 2ebb98f4..6228edef 100644
--- a/tests/a11y/runA11yTests.js
+++ b/tests/a11y/runA11yTests.js
@@ -14,7 +14,7 @@ const config = require( path.resolve( __dirname, 'a11y.config.js' ) );
  */
 function resetReportDir() {
 	// Delete and create report directory
-	fs.rmdirSync( config.reportDir, { recursive: true } );
+	fs.rmSync( config.reportDir, { recursive: true } );
 	fs.mkdirSync( config.reportDir, { recursive: true } );
 }
 
@@ -27,8 +27,8 @@ function resetReportDir() {
  * @return {Promise<any>}
  */
 function sendMetrics( namespace, name, count ) {
-	const metricPrefix = 'MediaWiki.a11y';
-	const url = `${process.env.BEACON_URL}${metricPrefix}.${namespace}.${name}=${count}c`;
+	const metricPrefix = 'ci_a11y';
+	const url = `${process.env.WMF_JENKINS_BEACON_URL}${metricPrefix}.${namespace}.${name}=${count}c`;
 	return fetch( url );
 }
 
@@ -38,6 +38,12 @@ function sendMetrics( namespace, name, count ) {
  * @param {Object} opts
  */
 async function runTests( opts ) {
+	if ( !process.env.MW_SERVER ||
+		!process.env.MEDIAWIKI_USER ||
+		!process.env.MEDIAWIKI_PASSWORD ) {
+		throw new Error( 'Missing env variables' );
+	}
+
 	try {
 		const tests = config.tests;
 		const allValidTests = tests.filter( ( test ) => test.name ).length === tests.length;
@@ -45,7 +51,7 @@ async function runTests( opts ) {
 			throw new Error( 'Config missing test name' );
 		}
 
-		const canLogResults = process.env.BEACON_URL && config.namespace;
+		const canLogResults = process.env.WMF_JENKINS_BEACON_URL && config.namespace;
 		if ( opts.logResults && !canLogResults ) {
 			throw new Error( 'Unable to log results, missing config or env variables' );
 		}
@@ -78,7 +84,7 @@ async function runTests( opts ) {
 			}
 
 			// Send data to Graphite
-			// BEACON_URL is only defined in CI env
+			// WMF_JENKINS_BEACON_URL is only defined in CI env
 			if ( opts.logResults && canLogResults ) {
 				await sendMetrics( config.namespace, testResult.name, errorNum )
 					.then( ( response ) => {