Sticky header AB test bucketing for 2 treatment buckets
For idwiki/viwiki, we wish to run the sticky header edit button AB
test so that treatment1 group sees the sticky header without edit
buttons, treatment2 groups sees the sticky header with edit buttons,
and the control/unsampled groups see no sticky header at all.
This patch overrides the configuration to make the sticky header
w/o edit buttons for treatment1, sticky header w/ edit buttons for
treatment2, and hides sticky header for everyone else. This depends
on a configuration with the treatment groups having "treatment1"
and "treatment2" as substrings in their bucket names.
The full configuration for idwiki/viwiki would be something like
the following:
```
$wgVectorStickyHeader = [
"logged_in" => true,
"logged_out" => false,
];
$wgVectorStickyHeaderEdit = [
"logged_in" => true,
"logged_out" => false,
];
$wgVectorWebABTestEnrollment = [
"name" => "vector.sticky_header_edit",
"enabled" => true,
"buckets" => [
"unsampled" => [
"samplingRate" => 0
],
"noStickyHeaderControl" => [
"samplingRate" => 0.34
],
"stickyHeaderNoEditButtonTreatment1" => [
"samplingRate" => 0.33
],
"stickyHeaderEditButtonTreatment2" => [
"samplingRate" => 0.33
]
],
];
```
Bug: T312573
Change-Id: I15c360fdf5393f5594602acc33b5b916e904016d
(cherry picked from commit 942cd5b0f6)
This commit is contained in:
Jan Drewniak
Clare Ming
parent
8772b0065e
commit
4c52b69780
|
|
@ -3,6 +3,11 @@
|
|||
const EXCLUDED_BUCKET = 'unsampled';
|
||||
const TREATMENT_BUCKET_SUBSTRING = 'treatment';
|
||||
const WEB_AB_TEST_ENROLLMENT_HOOK = 'mediawiki.web_AB_test_enrollment';
|
||||
/**
|
||||
* @typedef {Function} TreatmentBucketFunction
|
||||
* @param {string} [a]
|
||||
* @return {boolean}
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} WebABTest
|
||||
|
|
@ -10,7 +15,7 @@ const WEB_AB_TEST_ENROLLMENT_HOOK = 'mediawiki.web_AB_test_enrollment';
|
|||
* @property {function(): string} getBucket
|
||||
* @property {function(string): boolean} isInBucket
|
||||
* @property {function(): boolean} isInSample
|
||||
* @property {function(): boolean} isInTreatmentBucket
|
||||
* @property {TreatmentBucketFunction} isInTreatmentBucket
|
||||
*/
|
||||
|
||||
/**
|
||||
|
|
@ -41,12 +46,15 @@ const WEB_AB_TEST_ENROLLMENT_HOOK = 'mediawiki.web_AB_test_enrollment';
|
|||
* name: 'nameOfExperiment',
|
||||
* buckets: {
|
||||
* unsampled: {
|
||||
* samplingRate: 0.5
|
||||
* samplingRate: 0.25
|
||||
* },
|
||||
* control: {
|
||||
* samplingRate: 0.25
|
||||
* },
|
||||
* treatment: {
|
||||
* treatment1: {
|
||||
* samplingRate: 0.25
|
||||
* },
|
||||
* treatment2: {
|
||||
* samplingRate: 0.25
|
||||
* }
|
||||
* },
|
||||
|
|
@ -147,17 +155,18 @@ module.exports = function webABTest( props ) {
|
|||
|
||||
/**
|
||||
* Whether or not the subject has been bucketed in a treatment bucket as
|
||||
* defined by the bucket name containing the case-insensitive `treatment`
|
||||
* substring (e.g. 'treatment', 'sticky-header-treatment' and
|
||||
* 'stickyHeaderTreatment' are all assumed to be treatment buckets).
|
||||
* defined by the bucket name containing the case-insensitive 'treatment',
|
||||
* 'treatment1', or 'treatment2' substring (e.g. 'treatment', 'treatment1',
|
||||
* 'sticky-header-treatment1' and 'stickyHeaderTreatment2' are all assumed
|
||||
* to be treatment buckets).
|
||||
*
|
||||
* @param {string|null} [treatmentBucketName] lowercase name of bucket.
|
||||
* @return {boolean}
|
||||
*/
|
||||
function isInTreatmentBucket() {
|
||||
function isInTreatmentBucket( treatmentBucketName = '' ) {
|
||||
const bucket = getBucket();
|
||||
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
return bucket.toLowerCase().includes( TREATMENT_BUCKET_SUBSTRING );
|
||||
return bucket.toLowerCase().includes( `${TREATMENT_BUCKET_SUBSTRING}${treatmentBucketName}` );
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ const
|
|||
initTableOfContents = require( './tableOfContents.js' ),
|
||||
deferUntilFrame = require( './deferUntilFrame.js' ),
|
||||
ABTestConfig = require( /** @type {string} */ ( './config.json' ) ).wgVectorWebABTestEnrollment || {},
|
||||
stickyHeaderEditIconConfig = require( /** @type {string} */ ( './config.json' ) ).wgVectorStickyHeaderEdit || true,
|
||||
TOC_ID = 'mw-panel-toc',
|
||||
TOC_ID_LEGACY = 'toc',
|
||||
BODY_CONTENT_ID = 'bodyContent',
|
||||
|
|
@ -58,7 +59,7 @@ const getHeadingIntersectionHandler = ( changeActiveSection ) => {
|
|||
function initStickyHeaderABTests( abConfig, isStickyHeaderFeatureAllowed, getEnabledExperiment ) {
|
||||
let show = isStickyHeaderFeatureAllowed,
|
||||
stickyHeaderExperiment,
|
||||
noEditIcons = true;
|
||||
noEditIcons = stickyHeaderEditIconConfig;
|
||||
|
||||
// One of the sticky header AB tests is specified in the config
|
||||
const abTestName = abConfig.name,
|
||||
|
|
@ -74,16 +75,21 @@ function initStickyHeaderABTests( abConfig, isStickyHeaderFeatureAllowed, getEna
|
|||
// If eligible, initialize the AB test
|
||||
stickyHeaderExperiment = getEnabledExperiment( abConfig );
|
||||
|
||||
// If running initial AB test, only show sticky header to treatment group.
|
||||
if ( abTestName === stickyHeader.STICKY_HEADER_EXPERIMENT_NAME ) {
|
||||
// If running initial or edit AB test, show sticky header to treatment groups
|
||||
// only. Unsampled and control buckets do not see sticky header.
|
||||
if ( abTestName === stickyHeader.STICKY_HEADER_EXPERIMENT_NAME ||
|
||||
abTestName === stickyHeader.STICKY_HEADER_EDIT_EXPERIMENT_NAME
|
||||
) {
|
||||
show = stickyHeaderExperiment.isInTreatmentBucket();
|
||||
}
|
||||
|
||||
// If running edit-button AB test, show sticky header to all buckets
|
||||
// and show edit button for treatment group
|
||||
// If running edit-button AB test, the edit buttons in sticky header are shown
|
||||
// to second treatment group only.
|
||||
if ( abTestName === stickyHeader.STICKY_HEADER_EDIT_EXPERIMENT_NAME ) {
|
||||
show = true;
|
||||
if ( stickyHeaderExperiment.isInTreatmentBucket() ) {
|
||||
if ( stickyHeaderExperiment.isInTreatmentBucket( '1' ) ) {
|
||||
noEditIcons = true;
|
||||
}
|
||||
if ( stickyHeaderExperiment.isInTreatmentBucket( '2' ) ) {
|
||||
noEditIcons = false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ describe( 'main.js', () => {
|
|||
isEnabled: true,
|
||||
isUserInTreatmentBucket: false,
|
||||
expectedResult: {
|
||||
showStickyHeader: true,
|
||||
showStickyHeader: false,
|
||||
disableEditIcons: true
|
||||
}
|
||||
},
|
||||
|
|
@ -78,7 +78,7 @@ describe( 'main.js', () => {
|
|||
{
|
||||
abConfig: STICKY_HEADER_AB,
|
||||
isEnabled: false, // sticky header unavailable
|
||||
isUserInTreatmentBucket: false, // not in treament bucket
|
||||
isUserInTreatmentBucket: false, // not in treatment bucket
|
||||
expectedResult: {
|
||||
showStickyHeader: false,
|
||||
disableEditIcons: true
|
||||
|
|
@ -87,7 +87,7 @@ describe( 'main.js', () => {
|
|||
{
|
||||
abConfig: STICKY_HEADER_AB,
|
||||
isEnabled: true, // sticky header available
|
||||
isUserInTreatmentBucket: false, // not in treament bucket
|
||||
isUserInTreatmentBucket: false, // not in treatment bucket
|
||||
expectedResult: {
|
||||
showStickyHeader: false,
|
||||
disableEditIcons: true
|
||||
|
|
@ -96,7 +96,7 @@ describe( 'main.js', () => {
|
|||
{
|
||||
abConfig: STICKY_HEADER_AB,
|
||||
isEnabled: false, // sticky header is not available
|
||||
isUserInTreatmentBucket: true, // but the user is in the treament bucket
|
||||
isUserInTreatmentBucket: true, // but the user is in the treatment bucket
|
||||
expectedResult: {
|
||||
showStickyHeader: false,
|
||||
disableEditIcons: true
|
||||
|
|
@ -129,27 +129,33 @@ describe( 'main.js', () => {
|
|||
disableEditIcons: false
|
||||
}
|
||||
}
|
||||
].forEach( ( { abConfig, isEnabled, isUserInTreatmentBucket, expectedResult } ) => {
|
||||
document.documentElement.classList.add( 'vector-sticky-header-enabled' );
|
||||
const result = test.initStickyHeaderABTests(
|
||||
].forEach(
|
||||
( {
|
||||
abConfig,
|
||||
// isStickyHeaderFeatureAllowed
|
||||
isEnabled,
|
||||
( experiment ) => ( {
|
||||
name: experiment.name,
|
||||
isInBucket: () => true,
|
||||
isInSample: () => true,
|
||||
getBucket: () => 'bucket',
|
||||
isInTreatmentBucket: () => {
|
||||
return isUserInTreatmentBucket;
|
||||
}
|
||||
} )
|
||||
);
|
||||
expect( result ).toMatchObject( expectedResult );
|
||||
// Check that there are no side effects
|
||||
expect(
|
||||
document.documentElement.classList.contains( 'vector-sticky-header-enabled' )
|
||||
).toBe( true );
|
||||
} );
|
||||
isUserInTreatmentBucket,
|
||||
expectedResult
|
||||
} ) => {
|
||||
document.documentElement.classList.add( 'vector-sticky-header-enabled' );
|
||||
const result = test.initStickyHeaderABTests(
|
||||
abConfig,
|
||||
// isStickyHeaderFeatureAllowed
|
||||
isEnabled,
|
||||
( experiment ) => ( {
|
||||
name: experiment.name,
|
||||
isInBucket: () => true,
|
||||
isInSample: () => true,
|
||||
getBucket: () => 'bucket',
|
||||
isInTreatmentBucket: () => {
|
||||
return isUserInTreatmentBucket;
|
||||
}
|
||||
} )
|
||||
);
|
||||
expect( result ).toMatchObject( expectedResult );
|
||||
// Check that there are no side effects
|
||||
expect(
|
||||
document.documentElement.classList.contains( 'vector-sticky-header-enabled' )
|
||||
).toBe( true );
|
||||
} );
|
||||
} );
|
||||
} );
|
||||
|
|
|
|||
Loading…
Reference in New Issue