Change robots to deny by default with allowlist

app/controllers/robots_controller.rb

@@ -0,0 +1,25 @@
+class RobotsController < ApplicationController
+  ALLOWED_ROUTES = %i[
+    new_user_session
+    forgot_password
+    sign_up_email
+  ].to_set.freeze
+
+  def index
+    render plain: [
+      'User-agent: *',
+      'Disallow: /',
+      *allowed_paths.map { |path| "Allow: #{path}$" },
+    ].join("\n")
+  end
+
+  private
+
+  def allowed_paths
+    I18n.available_locales.
+      map { |locale| locale == I18n.default_locale ? nil : locale }.
+      flat_map do |locale|
+        ALLOWED_ROUTES.map { |route| route_for(route, only_path: true, locale:) }
+      end
+  end
+end

app/views/layouts/base.html.erb

@@ -11,10 +11,6 @@
   <meta content="<%= yield(:meta_refresh) %>" http-equiv="refresh" />
   <% end %>
 
-  <% if session_with_trust? || FeatureManagement.disallow_all_web_crawlers? %>
-  <meta content="noindex,nofollow" name="robots" />
-  <% end %>
-
   <title><%= title %> | <%= APP_NAME %></title>
 
   <%= javascript_tag(nonce: true) do %>

config/application.yml.default

@@ -71,7 +71,6 @@ deleted_user_accounts_report_configs: '[]'
 development_mailer_deliver_method: letter_opener
 disable_email_sending: true
 disable_logout_get_request: true
-disallow_all_web_crawlers: true
 disposable_email_services: '[]'
 doc_auth_attempt_window_in_minutes: 360
 doc_capture_polling_enabled: true
@@ -457,7 +456,6 @@ production:
   database_worker_jobs_password: ''
   disable_email_sending: false
   disable_logout_get_request: false
-  disallow_all_web_crawlers: false
   doc_auth_vendor: 'acuant'
   doc_auth_vendor_randomize: false
   doc_auth_vendor_randomize_percent: 0

config/routes.rb

@@ -56,6 +56,7 @@
   get '/openid_connect/logout' => 'openid_connect/logout#index'
   delete '/openid_connect/logout' => 'openid_connect/logout#delete'
 
+  get '/robots.txt' => 'robots#index'
   get '/no_js/detect.css' => 'no_js#index', as: :no_js_detect_css
 
   # i18n routes. Alphabetically sorted.

lib/feature_management.rb

@@ -76,10 +76,6 @@ def self.enable_saml_cert_rotation?
     IdentityConfig.store.saml_secret_rotation_enabled
   end
 
-  def self.disallow_all_web_crawlers?
-    IdentityConfig.store.disallow_all_web_crawlers
-  end
-
   def self.gpo_upload_enabled?
     # leaving the usps name for backwards compatibility
     IdentityConfig.store.usps_upload_enabled

lib/identity_config.rb

@@ -171,7 +171,6 @@ def self.build_store(config_map)
     config.add(:development_mailer_deliver_method, type: :symbol, enum: [:file, :letter_opener])
     config.add(:disable_email_sending, type: :boolean)
     config.add(:disable_logout_get_request, type: :boolean)
-    config.add(:disallow_all_web_crawlers, type: :boolean)
     config.add(:disposable_email_services, type: :json)
     config.add(:doc_auth_attempt_window_in_minutes, type: :integer)
     config.add(:doc_auth_check_failed_image_resubmission_enabled, type: :boolean)

spec/controllers/robots_controller_spec.rb

@@ -0,0 +1,28 @@
+require 'rails_helper'
+
+RSpec.describe RobotsController do
+  describe '#index' do
+    subject(:response) { get :index }
+    let(:lines) { response.body.lines(chomp: true) }
+
+    it 'renders plaintext' do
+      expect(response.content_type.split(';').first).to eq('text/plain')
+    end
+
+    it 'targets all crawlers' do
+      expect(lines).to include('User-agent: *')
+    end
+
+    it 'denies all by default' do
+      expect(lines).to include('Disallow: /')
+    end
+
+    it 'allows public routes' do
+      expect(lines).to include('Allow: /$')
+    end
+
+    it 'allows localized version of public routes' do
+      expect(lines).to include('Allow: /es$')
+    end
+  end
+end

spec/controllers/sign_up/passwords_controller_spec.rb

@@ -169,13 +169,6 @@
 
   describe '#new' do
     render_views
-    it 'instructs crawlers to not index this page' do
-      token = 'foo token'
-      create(:user, :unconfirmed, confirmation_token: token)
-      get :new, params: { confirmation_token: token }
-
-      expect(response.body).to match('<meta content="noindex,nofollow" name="robots" />')
-    end
 
     it 'rejects when confirmation_token is invalid' do
       invalid_confirmation_sent_at =

spec/controllers/users/reset_passwords_controller_spec.rb

@@ -141,7 +141,7 @@
           allow(user).to receive(:email_addresses).and_return([email_address])
         end
 
-        it 'displays the form to enter a new password and disallows indexing' do
+        it 'displays the form to enter a new password' do
           expect(email_address).to receive(:email).twice
 
           forbidden = instance_double(ForbiddenPasswords)
@@ -156,7 +156,6 @@
 
           expect(response).to render_template :edit
           expect(flash.keys).to be_empty
-          expect(response.body).to match('<meta content="noindex,nofollow" name="robots" />')
         end
       end
     end

spec/lib/feature_management_spec.rb

@@ -270,20 +270,6 @@
     end
   end
 
-  describe '#disallow_all_web_crawlers?' do
-    it 'returns true when IdentityConfig setting is true' do
-      allow(IdentityConfig.store).to receive(:disallow_all_web_crawlers) { true }
-
-      expect(FeatureManagement.disallow_all_web_crawlers?).to eq(true)
-    end
-
-    it 'returns false when IdentityConfig setting is false' do
-      allow(IdentityConfig.store).to receive(:disallow_all_web_crawlers) { false }
-
-      expect(FeatureManagement.disallow_all_web_crawlers?).to eq(false)
-    end
-  end
-
   describe '#identity_pki_local_dev?' do
     context 'when in development mode' do
       before(:each) do