jonny
/
Vector-Local
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sticky header AB test bucketing for 2 treatment buckets 

For idwiki/viwiki, we wish to run the sticky header edit button AB
test so that treatment1 group sees the sticky header without edit
buttons, treatment2 groups sees the sticky header with edit buttons,
and the control/unsampled groups see no sticky header at all.

This patch overrides the configuration to make the sticky header
w/o edit buttons for treatment1, sticky header w/ edit buttons for
treatment2, and hides sticky header for everyone else. This depends
on a configuration with the treatment groups having "treatment1"
and "treatment2" as substrings in their bucket names.

The full configuration for idwiki/viwiki would be something like
the following:

```
$wgVectorStickyHeader = [
	"logged_in" =>  true,
	"logged_out" => false,
];

$wgVectorStickyHeaderEdit = [
		"logged_in" => true,
		"logged_out" => false,
];

$wgVectorWebABTestEnrollment = [
	"name" => "vector.sticky_header_edit",
	"enabled" => true,
	"buckets" => [
		"unsampled" => [
			"samplingRate" => 0
		],
                "noStickyHeaderControl" => [
                        "samplingRate" => 0.34
                ],
		"stickyHeaderNoEditButtonTreatment1" => [
			"samplingRate" => 0.33
		],
		"stickyHeaderEditButtonTreatment2" => [
			"samplingRate" => 0.33
		]
	],
];
```

Bug: T312573
Change-Id: I15c360fdf5393f5594602acc33b5b916e904016d
This commit is contained in:
Jan Drewniak 2022-07-07 15:06:14 -04:00 committed by Jdlrobson
parent d4d50d2dc5
commit 942cd5b0f6
3 changed files with 61 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
resources/skins.vector.es6/AB.js
View File

@ -3,6 +3,11 @@
const EXCLUDED_BUCKET = 'unsampled';
const TREATMENT_BUCKET_SUBSTRING = 'treatment';
const WEB_AB_TEST_ENROLLMENT_HOOK = 'mediawiki.web_AB_test_enrollment';
/**
* @typedef {Function} TreatmentBucketFunction
* @param {string} [a]
* @return {boolean}
*/
/**
* @typedef {Object} WebABTest
@ -10,7 +15,7 @@ const WEB_AB_TEST_ENROLLMENT_HOOK = 'mediawiki.web_AB_test_enrollment';
* @property {function(): string} getBucket
* @property {function(string): boolean} isInBucket
* @property {function(): boolean} isInSample
* @property {function(): boolean} isInTreatmentBucket
* @property {TreatmentBucketFunction} isInTreatmentBucket
*/
/**
@ -41,12 +46,15 @@ const WEB_AB_TEST_ENROLLMENT_HOOK = 'mediawiki.web_AB_test_enrollment';
* name: 'nameOfExperiment',
* buckets: {
* unsampled: {
* samplingRate: 0.5
* samplingRate: 0.25
* },
* control: {
* samplingRate: 0.25
* },
* treatment: {
* treatment1: {
* samplingRate: 0.25
* },
* treatment2: {
* samplingRate: 0.25
* }
* },
@ -147,17 +155,18 @@ module.exports = function webABTest( props ) {
/**
* Whether or not the subject has been bucketed in a treatment bucket as
* defined by the bucket name containing the case-insensitive `treatment`
* substring (e.g. 'treatment', 'sticky-header-treatment' and
* 'stickyHeaderTreatment' are all assumed to be treatment buckets).
* defined by the bucket name containing the case-insensitive 'treatment',
* 'treatment1', or 'treatment2' substring (e.g. 'treatment', 'treatment1',
* 'sticky-header-treatment1' and 'stickyHeaderTreatment2' are all assumed
* to be treatment buckets).
*
* @param {string|null} [treatmentBucketName] lowercase name of bucket.
* @return {boolean}
*/
function isInTreatmentBucket() {
function isInTreatmentBucket( treatmentBucketName = '' ) {
const bucket = getBucket();
// eslint-disable-next-line no-restricted-syntax
return bucket.toLowerCase().includes( TREATMENT_BUCKET_SUBSTRING );
return bucket.toLowerCase().includes( `${TREATMENT_BUCKET_SUBSTRING}${treatmentBucketName}` );
}
/**

20
resources/skins.vector.es6/main.js
View File

@ -8,6 +8,7 @@ const
initTableOfContents = require( './tableOfContents.js' ),
deferUntilFrame = require( './deferUntilFrame.js' ),
ABTestConfig = require( /** @type {string} */ ( './config.json' ) ).wgVectorWebABTestEnrollment || {},
stickyHeaderEditIconConfig = require( /** @type {string} */ ( './config.json' ) ).wgVectorStickyHeaderEdit || true,
TOC_ID = 'mw-panel-toc',
TOC_ID_LEGACY = 'toc',
BODY_CONTENT_ID = 'bodyContent',
@ -58,7 +59,7 @@ const getHeadingIntersectionHandler = ( changeActiveSection ) => {
function initStickyHeaderABTests( abConfig, isStickyHeaderFeatureAllowed, getEnabledExperiment ) {
let show = isStickyHeaderFeatureAllowed,
stickyHeaderExperiment,
noEditIcons = true;
noEditIcons = stickyHeaderEditIconConfig;
// One of the sticky header AB tests is specified in the config
const abTestName = abConfig.name,
@ -74,16 +75,21 @@ function initStickyHeaderABTests( abConfig, isStickyHeaderFeatureAllowed, getEna
// If eligible, initialize the AB test
stickyHeaderExperiment = getEnabledExperiment( abConfig );
// If running initial AB test, only show sticky header to treatment group.
if ( abTestName === stickyHeader.STICKY_HEADER_EXPERIMENT_NAME ) {
// If running initial or edit AB test, show sticky header to treatment groups
// only. Unsampled and control buckets do not see sticky header.
if ( abTestName === stickyHeader.STICKY_HEADER_EXPERIMENT_NAME ||
abTestName === stickyHeader.STICKY_HEADER_EDIT_EXPERIMENT_NAME
) {
show = stickyHeaderExperiment.isInTreatmentBucket();
}
// If running edit-button AB test, show sticky header to all buckets
// and show edit button for treatment group
// If running edit-button AB test, the edit buttons in sticky header are shown
// to second treatment group only.
if ( abTestName === stickyHeader.STICKY_HEADER_EDIT_EXPERIMENT_NAME ) {
show = true;
if ( stickyHeaderExperiment.isInTreatmentBucket() ) {
if ( stickyHeaderExperiment.isInTreatmentBucket( '1' ) ) {
noEditIcons = true;
}
if ( stickyHeaderExperiment.isInTreatmentBucket( '2' ) ) {
noEditIcons = false;
}
}

54
tests/jest/skins.vector.es6/main.test.js
View File

@ -62,7 +62,7 @@ describe( 'main.js', () => {
isEnabled: true,
isUserInTreatmentBucket: false,
expectedResult: {
showStickyHeader: true,
showStickyHeader: false,
disableEditIcons: true
}
},
@ -78,7 +78,7 @@ describe( 'main.js', () => {
{
abConfig: STICKY_HEADER_AB,
isEnabled: false, // sticky header unavailable
isUserInTreatmentBucket: false, // not in treament bucket
isUserInTreatmentBucket: false, // not in treatment bucket
expectedResult: {
showStickyHeader: false,
disableEditIcons: true
@ -87,7 +87,7 @@ describe( 'main.js', () => {
{
abConfig: STICKY_HEADER_AB,
isEnabled: true, // sticky header available
isUserInTreatmentBucket: false, // not in treament bucket
isUserInTreatmentBucket: false, // not in treatment bucket
expectedResult: {
showStickyHeader: false,
disableEditIcons: true
@ -96,7 +96,7 @@ describe( 'main.js', () => {
{
abConfig: STICKY_HEADER_AB,
isEnabled: false, // sticky header is not available
isUserInTreatmentBucket: true, // but the user is in the treament bucket
isUserInTreatmentBucket: true, // but the user is in the treatment bucket
expectedResult: {
showStickyHeader: false,
disableEditIcons: true
@ -129,27 +129,33 @@ describe( 'main.js', () => {
disableEditIcons: false
}
}
].forEach( ( { abConfig, isEnabled, isUserInTreatmentBucket, expectedResult } ) => {
document.documentElement.classList.add( 'vector-sticky-header-enabled' );
const result = test.initStickyHeaderABTests(
].forEach(
( {
abConfig,
// isStickyHeaderFeatureAllowed
isEnabled,
( experiment ) => ( {
name: experiment.name,
isInBucket: () => true,
isInSample: () => true,
getBucket: () => 'bucket',
isInTreatmentBucket: () => {
return isUserInTreatmentBucket;
}
} )
);
expect( result ).toMatchObject( expectedResult );
// Check that there are no side effects
expect(
document.documentElement.classList.contains( 'vector-sticky-header-enabled' )
).toBe( true );
} );
isUserInTreatmentBucket,
expectedResult
} ) => {
document.documentElement.classList.add( 'vector-sticky-header-enabled' );
const result = test.initStickyHeaderABTests(
abConfig,
// isStickyHeaderFeatureAllowed
isEnabled,
( experiment ) => ( {
name: experiment.name,
isInBucket: () => true,
isInSample: () => true,
getBucket: () => 'bucket',
isInTreatmentBucket: () => {
return isUserInTreatmentBucket;
}
} )
);
expect( result ).toMatchObject( expectedResult );
// Check that there are no side effects
expect(
document.documentElement.classList.contains( 'vector-sticky-header-enabled' )
).toBe( true );
} );
} );
} );