LG-9449: Prevent Location Search slowdown and errors from ArcGIS auth

app/jobs/arcgis_token_job.rb

@@ -0,0 +1,27 @@
+class ArcgisTokenJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    analytics.idv_arcgis_token_job_started
+    token_entry = token_keeper.retrieve_token
+    token_keeper.save_token(token_entry, token_entry.expires_at)
+    return true
+  ensure
+    analytics.idv_arcgis_token_job_completed
+  end
+
+  private
+
+  def token_keeper
+    ArcgisApi::TokenKeeper.new
+  end
+
+  def analytics
+    @analytics ||= Analytics.new(
+      user: AnonymousUser.new,
+      request: nil,
+      session: {},
+      sp: nil,
+    )
+  end
+end

app/services/analytics_events.rb

@@ -559,6 +559,45 @@ def idv_arcgis_request_failure(
     )
   end
 
+  # Tracks if request to get auth token from ArcGIS fails
+  # @param [String] exception_class
+  # @param [String] exception_message
+  # @param [Boolean] response_body_present
+  # @param [Hash] response_body
+  # @param [Integer] response_status_code
+  def idv_arcgis_token_failure(
+    exception_class:,
+    exception_message:,
+    response_body_present:,
+    response_body:,
+    response_status_code:,
+    **extra
+  )
+    track_event(
+      'Request ArcGIS Token: request failed',
+      exception_class: exception_class,
+      exception_message: exception_message,
+      response_body_present: response_body_present,
+      response_body: response_body,
+      response_status_code: response_status_code,
+      **extra,
+    )
+  end
+
+  # Track when token job ended
+  def idv_arcgis_token_job_completed
+    track_event(
+      'ArcgisTokenJob: Completed',
+    )
+  end
+
+  # Track when token job started
+  def idv_arcgis_token_job_started
+    track_event(
+      'ArcgisTokenJob: Started',
+    )
+  end
+
   # @param [String] step the step that the user was on when they clicked cancel
   # @param [Idv::ProofingComponentsLogging] proofing_components User's current proofing components
   # The user confirmed their choice to cancel going through IDV

app/services/arcgis_api/connection_factory.rb

@@ -0,0 +1,22 @@
+module ArcgisApi
+  # Stateless and thread-safe factory object that create a new Faraday::Connection object.
+  class ConnectionFactory
+    # @param [String|URI] url
+    # @options [Hash] Faraday connection options
+    # @return Faraday::Connection
+    def connection(url = nil, options = {})
+      conn_options = options ? options.dup : {}
+      Faraday.new(url, conn_options) do |conn|
+        # Log request metrics
+        conn.request :instrumentation, name: 'request_metric.faraday'
+        conn.options.timeout = IdentityConfig.store.arcgis_api_request_timeout_seconds
+        # Parse JSON responses
+        conn.response :json, content_type: 'application/json'
+        # Raise an error subclassing Faraday::Error on 4xx, 5xx, and malformed responses
+        # Note: The order of this matters for parsing the error response body.
+        conn.response :raise_error
+        yield conn if block_given?
+      end
+    end
+  end
+end

app/services/arcgis_api/geocoder.rb

@@ -6,8 +6,6 @@ class Geocoder
       keyword_init: true
     )
     Location = Struct.new(:latitude, :longitude, keyword_init: true)
-    API_TOKEN_HOST = URI(IdentityConfig.store.arcgis_api_generate_token_url).host
-    API_TOKEN_CACHE_KEY = "arcgis_api_token:#{API_TOKEN_HOST}"
 
     # These are option URL params that tend to apply to multiple endpoints
     # https://developers.arcgis.com/rest/geocode/api-reference/geocoding-find-address-candidates.htm#ESRI_SECTION2_38613C3FCB12462CAADD55B2905140BF
@@ -41,24 +39,10 @@ class Geocoder
       :SingleLine, # Unvalidated address-like text string used to search for geocoded addresses
     ]
 
-    # Makes an HTTP request to quickly find potential address matches. Each match that is found
-    # will include an associated magic_key value which can later be used to get more details about
-    # the address using the #find_address_candidates method
-    # Requests text input and will only match possible addresses
-    # A maximum of 5 suggestions are included in the suggestions array.
-    # @param text [String]
-    # @return [Array<Suggestion>] Suggestions
-    def suggest(text)
-      params = {
-        text: text,
-        **COMMON_DEFAULT_PARAMETERS,
-      }
+    private attr_accessor :token_keeper
 
-      parse_suggestions(
-        faraday.get(IdentityConfig.store.arcgis_api_suggest_url, params, dynamic_headers) do |req|
-          req.options.context = { service_name: 'arcgis_geocoder_suggest' }
-        end.body,
-      )
+    def initialize(token_keeper: TokenKeeper.new)
+      @token_keeper = token_keeper
     end
 
     # Makes HTTP request to find a full address record using a magic key or single text line
@@ -83,68 +67,34 @@ def find_address_candidates(**options)
       }
 
       parse_address_candidates(
-        faraday.get(
-          IdentityConfig.store.arcgis_api_find_address_candidates_url, params,
-          dynamic_headers
+        connection.get(
+          IdentityConfig.store.arcgis_api_find_address_candidates_url, params, dynamic_headers
         ) do |req|
           req.options.context = { service_name: 'arcgis_geocoder_find_address_candidates' }
         end.body,
       )
     end
 
-    # Makes a request to retrieve a new token
-    # it expires after 1 hour
-    # @return [String] Auth token
-    def retrieve_token!
-      token, expires = request_token.fetch_values('token', 'expires')
-      expires_at = Time.zone.at(expires / 1000)
-      Rails.cache.write(API_TOKEN_CACHE_KEY, token, expires_at: expires_at)
-      # If using a redis cache we have to manually set the expires_at. This is because we aren't
-      # using a dedicated Redis cache and instead are just using our existing Redis server with
-      # mixed usage patterns. Without this cache entries don't expire.
-      # More at https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html
-      Rails.cache.try(:redis)&.expireat(API_TOKEN_CACHE_KEY, expires_at.to_i)
-      token
-    end
-
-    # Checks the cache for an unexpired token and returns that.
-    # If the cache has expired, retrieves a new token and returns it
-    # @return [String] Auth token
     def token
-      Rails.cache.read(API_TOKEN_CACHE_KEY) || retrieve_token!
+      token_keeper&.token
     end
 
     private
 
-    def faraday
+    def connection
       Faraday.new do |conn|
         # Log request metrics
         conn.request :instrumentation, name: 'request_metric.faraday'
-
         conn.options.timeout = IdentityConfig.store.arcgis_api_request_timeout_seconds
-
         # Raise an error subclassing Faraday::Error on 4xx, 5xx, and malformed responses
         # Note: The order of this matters for parsing the error response body.
         conn.response :raise_error
-
         # Parse JSON responses
         conn.response :json, content_type: 'application/json'
-
         yield conn if block_given?
       end
     end
 
-    def parse_suggestions(response_body)
-      handle_api_errors(response_body)
-
-      response_body['suggestions'].map do |suggestion|
-        Suggestion.new(
-          text: suggestion['text'],
-          magic_key: suggestion['magicKey'],
-        )
-      end
-    end
-
     def parse_address_candidates(response_body)
       handle_api_errors(response_body)
 
@@ -180,7 +130,6 @@ def handle_api_errors(response_body)
           response_status_code: error_code,
           api_status_code: error_code,
         )
-        Rails.cache.delete(API_TOKEN_CACHE_KEY) # this might only be needed for local testing
         raise Faraday::ClientError.new(
           RuntimeError.new(error_message),
           {
@@ -199,26 +148,6 @@ def dynamic_headers
       { 'Authorization' => "Bearer #{token}" }
     end
 
-    # Makes HTTP request to authentication endpoint and
-    # returns the token and when it expires (1 hour).
-    # @return [Hash] API response
-    def request_token
-      body = {
-        username: IdentityConfig.store.arcgis_api_username,
-        password: IdentityConfig.store.arcgis_api_password,
-        referer: IdentityConfig.store.domain_name,
-        f: 'json',
-      }
-
-      faraday.post(
-        IdentityConfig.store.arcgis_api_generate_token_url, URI.encode_www_form(body)
-      ) do |req|
-        req.options.context = { service_name: 'arcgis_token' }
-      end.body.tap do |body|
-        handle_api_errors(body)
-      end
-    end
-
     def analytics(user: AnonymousUser.new)
       Analytics.new(user: user, request: nil, session: {}, sp: nil)
     end

app/services/arcgis_api/mock/geocoder.rb

@@ -1,13 +1,56 @@
 module ArcgisApi
   module Mock
+    class TokenKeeper < ArcgisApi::TokenKeeper
+      def connection
+        stubs = Faraday::Adapter::Test::Stubs.new do |stub|
+          stub_generate_token(stub)
+        end
+        super do |con|
+          con.adapter :test, stubs
+        end
+      end
+
+      private
+
+      def stub_generate_token(stub)
+        stub.post(IdentityConfig.store.arcgis_api_generate_token_url) do |env|
+          [
+            200,
+            { 'Content-Type': 'application/json' },
+            {
+              token: '1234',
+              expires: (Time.zone.now + 1.minute).to_f * 1000,
+              ssl: true,
+            }.to_json,
+          ]
+        end
+      end
+    end
+
     class Geocoder < ArcgisApi::Geocoder
-      def faraday
-        super do |conn|
-          conn.adapter :test do |stub|
-            stub_generate_token(stub)
-            stub_suggestions(stub)
-            stub_address_candidates(stub)
-          end
+      # def faraday
+      #   super do |conn|
+      #     conn.adapter :test do |stub|
+      #       stub_generate_token(stub)
+      #       stub_suggestions(stub)
+      #       stub_address_candidates(stub)
+      #     end
+      #   end
+      # end
+
+      def initialize
+        token_keeper = TokenKeeper.new
+        super(token_keeper: token_keeper)
+      end
+
+      def connection
+        stubs = Faraday::Adapter::Test::Stubs.new do |stub|
+          stub_generate_token(stub)
+          stub_suggestions(stub)
+          stub_address_candidates(stub)
+        end
+        super do |con|
+          con.adapter :test, stubs
         end
       end
 
@@ -20,7 +63,7 @@ def stub_generate_token(stub)
             { 'Content-Type': 'application/json' },
             {
               token: '1234',
-              expires: 1234,
+              expires: (Time.zone.now + 30.seconds).to_f * 1000,
               ssl: true,
             }.to_json,
           ]

app/services/arcgis_api/response_validation.rb

@@ -0,0 +1,26 @@
+module ArcgisApi
+  # Faraday middleware to raise exception when response status is 200 but with error in body
+  class ResponseValidation < Faraday::Middleware
+    def on_complete(env)
+      return unless env[:status] == 200 && env.body
+      body = env.body.is_a?(String) ? JSON.parse(env.body) : env.body
+      return unless body.fetch('error', false)
+      handle_api_errors(body)
+    end
+
+    def handle_api_errors(response_body)
+      # response_body is in this format:
+      # {"error"=>{"code"=>400, "message"=>"", "details"=>[""]}}
+      error_code = response_body.dig('error', 'code')
+      error_message = response_body.dig('error', 'message') || "Received error code #{error_code}"
+      # log an error
+      raise ArcgisApi::InvalidResponseError.new(
+        Faraday::Error.new(error_message),
+        {
+          status: error_code,
+          body: { details: response_body.dig('error', 'details')&.join(', ') },
+        },
+      )
+    end
+  end
+end