From ea2819b443ca39ea2b550b6cd4cd07ccffc96560 Mon Sep 17 00:00:00 2001 From: Karl Baker Date: Tue, 12 Mar 2019 13:58:51 +0000 Subject: [PATCH] Add additional variant for Related Links A/B test v3 This commit adds an additional `C` variant for the upcoming related links A/B test iteration #3. In the previous test we saw some users who had the control variant cookie, yet navigated via the related links which are only shown on the `B` variant. In order to check if there are any problems with our A/B testing framework, in addition to making the data collection easier for later analysis, we are introducing the new `C` variant which will become our test variant. How the variants now work are as follows: - `A` variant: No users should ever end up in this variant, as our CDN will be configured it to assign 0% of users. If users do end up in this variant, we may have issues with our A/B testing framework which will require further investigation. - `B` variant: Our control variant, where related links will not be changed in any way. - `C` variant: Our test variant, where related links will be changed to use our generated links for the sample involved. --- app/controllers/concerns/ab_testable.rb | 2 +- app/controllers/content_items_controller.rb | 2 +- .../controllers/content_items_controller_test.rb | 16 +++++++++++++--- test/controllers/development_controller_test.rb | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/app/controllers/concerns/ab_testable.rb b/app/controllers/concerns/ab_testable.rb index 711800025..c52703bac 100644 --- a/app/controllers/concerns/ab_testable.rb +++ b/app/controllers/concerns/ab_testable.rb @@ -18,7 +18,7 @@ def related_links_test @related_links_test ||= GovukAbTesting::AbTest.new( "RelatedLinksABTest3", dimension: RELATED_LINKS_DIMENSION, - allowed_variants: %w(A B), + allowed_variants: %w(A B C), control_variant: "A" ) end diff --git a/app/controllers/content_items_controller.rb b/app/controllers/content_items_controller.rb index 0c74bb87f..c250a593b 100644 --- a/app/controllers/content_items_controller.rb +++ b/app/controllers/content_items_controller.rb @@ -57,7 +57,7 @@ def set_guide_draft_access_token def load_content_item content_item = Services.content_store.content_item(content_item_path) - if related_links_variant.variant?('B') && content_item.dig('links') + if related_links_variant.variant?('C') && content_item.dig('links') content_item['links']['ordered_related_items'] = content_item['links'].fetch('suggested_ordered_related_items', []) end diff --git a/test/controllers/content_items_controller_test.rb b/test/controllers/content_items_controller_test.rb index 08fb8e1db..5b4351143 100644 --- a/test/controllers/content_items_controller_test.rb +++ b/test/controllers/content_items_controller_test.rb @@ -130,7 +130,7 @@ class ContentItemsControllerTest < ActionController::TestCase assert_equal content_item['title'], assigns[:content_item].title end - test "gets item from the content store and keeps ordered_related_items when running RelatedLinksABTest3 control variant" do + test "gets item from the content store and keeps ordered_related_items when running RelatedLinksABTest3 misclassification variant" do with_variant RelatedLinksABTest3: 'A' do content_item = content_store_has_schema_example('case_study', 'case_study') @@ -140,10 +140,20 @@ class ContentItemsControllerTest < ActionController::TestCase end end - test "gets item from the content store and replaces ordered_related_items when running RelatedLinksABTest3 test variant" do + test "gets item from the content store and keeps ordered_related_items when running RelatedLinksABTest3 control variant" do with_variant RelatedLinksABTest3: 'B' do content_item = content_store_has_schema_example('case_study', 'case_study') + get :show, params: { path: path_for(content_item) } + assert_response :success + assert_equal content_item['links']['ordered_related_items'], assigns[:content_item].content_item['links']['ordered_related_items'] + end + end + + test "gets item from the content store and replaces ordered_related_items when running RelatedLinksABTest3 test variant" do + with_variant RelatedLinksABTest3: 'C' do + content_item = content_store_has_schema_example('case_study', 'case_study') + get :show, params: { path: path_for(content_item) } assert_response :success assert_equal assigns[:content_item].content_item['links']['ordered_related_items'], assigns[:content_item].content_item['links']['suggested_ordered_related_items'] @@ -151,7 +161,7 @@ class ContentItemsControllerTest < ActionController::TestCase end test "gets item from the content store and replaces ordered_related_items when empty array when RelatedLinksABTest3 test variant has no suggestions" do - with_variant RelatedLinksABTest3: 'B' do + with_variant RelatedLinksABTest3: 'C' do content_item = content_store_has_schema_example('guide', 'guide') get :show, params: { path: path_for(content_item) } diff --git a/test/controllers/development_controller_test.rb b/test/controllers/development_controller_test.rb index feed11ae5..347e60616 100644 --- a/test/controllers/development_controller_test.rb +++ b/test/controllers/development_controller_test.rb @@ -3,7 +3,7 @@ class DevelopmentControllerTest < ActionController::TestCase include GovukAbTesting::MinitestHelpers - %w(A B).each do |test_variant| + %w(A B C).each do |test_variant| test "RelatedLinksABTest3 works correctly for each variant (variant: #{test_variant})" do with_variant RelatedLinksABTest3: test_variant do get :index