diff --git a/app/jobs/arcgis_token_job.rb b/app/jobs/arcgis_token_job.rb new file mode 100644 index 00000000000..74dde5855eb --- /dev/null +++ b/app/jobs/arcgis_token_job.rb @@ -0,0 +1,27 @@ +class ArcgisTokenJob < ApplicationJob + queue_as :default + + def perform + analytics.idv_arcgis_token_job_started + token_entry = token_keeper.retrieve_token + token_keeper.save_token(token_entry, token_entry.expires_at) + return true + ensure + analytics.idv_arcgis_token_job_completed + end + + private + + def token_keeper + ArcgisApi::TokenKeeper.new + end + + def analytics + @analytics ||= Analytics.new( + user: AnonymousUser.new, + request: nil, + session: {}, + sp: nil, + ) + end +end diff --git a/app/services/analytics_events.rb b/app/services/analytics_events.rb index 58151b2599c..76c6b0bc313 100644 --- a/app/services/analytics_events.rb +++ b/app/services/analytics_events.rb @@ -559,6 +559,45 @@ def idv_arcgis_request_failure( ) end + # Tracks if request to get auth token from ArcGIS fails + # @param [String] exception_class + # @param [String] exception_message + # @param [Boolean] response_body_present + # @param [Hash] response_body + # @param [Integer] response_status_code + def idv_arcgis_token_failure( + exception_class:, + exception_message:, + response_body_present:, + response_body:, + response_status_code:, + **extra + ) + track_event( + 'Request ArcGIS Token: request failed', + exception_class: exception_class, + exception_message: exception_message, + response_body_present: response_body_present, + response_body: response_body, + response_status_code: response_status_code, + **extra, + ) + end + + # Track when token job ended + def idv_arcgis_token_job_completed + track_event( + 'ArcgisTokenJob: Completed', + ) + end + + # Track when token job started + def idv_arcgis_token_job_started + track_event( + 'ArcgisTokenJob: Started', + ) + end + # @param [String] step the step that the user was on when they clicked cancel # @param [Idv::ProofingComponentsLogging] proofing_components User's current proofing components # The user confirmed their choice to cancel going through IDV diff --git a/app/services/arcgis_api/connection_factory.rb b/app/services/arcgis_api/connection_factory.rb new file mode 100644 index 00000000000..a9a4484d804 --- /dev/null +++ b/app/services/arcgis_api/connection_factory.rb @@ -0,0 +1,22 @@ +module ArcgisApi + # Stateless and thread-safe factory object that create a new Faraday::Connection object. + class ConnectionFactory + # @param [String|URI] url + # @options [Hash] Faraday connection options + # @return Faraday::Connection + def connection(url = nil, options = {}) + conn_options = options ? options.dup : {} + Faraday.new(url, conn_options) do |conn| + # Log request metrics + conn.request :instrumentation, name: 'request_metric.faraday' + conn.options.timeout = IdentityConfig.store.arcgis_api_request_timeout_seconds + # Parse JSON responses + conn.response :json, content_type: 'application/json' + # Raise an error subclassing Faraday::Error on 4xx, 5xx, and malformed responses + # Note: The order of this matters for parsing the error response body. + conn.response :raise_error + yield conn if block_given? + end + end + end +end diff --git a/app/services/arcgis_api/geocoder.rb b/app/services/arcgis_api/geocoder.rb index 8976beece9b..22ba58f8ded 100644 --- a/app/services/arcgis_api/geocoder.rb +++ b/app/services/arcgis_api/geocoder.rb @@ -6,8 +6,6 @@ class Geocoder keyword_init: true ) Location = Struct.new(:latitude, :longitude, keyword_init: true) - API_TOKEN_HOST = URI(IdentityConfig.store.arcgis_api_generate_token_url).host - API_TOKEN_CACHE_KEY = "arcgis_api_token:#{API_TOKEN_HOST}" # These are option URL params that tend to apply to multiple endpoints # https://developers.arcgis.com/rest/geocode/api-reference/geocoding-find-address-candidates.htm#ESRI_SECTION2_38613C3FCB12462CAADD55B2905140BF @@ -41,24 +39,10 @@ class Geocoder :SingleLine, # Unvalidated address-like text string used to search for geocoded addresses ] - # Makes an HTTP request to quickly find potential address matches. Each match that is found - # will include an associated magic_key value which can later be used to get more details about - # the address using the #find_address_candidates method - # Requests text input and will only match possible addresses - # A maximum of 5 suggestions are included in the suggestions array. - # @param text [String] - # @return [Array] Suggestions - def suggest(text) - params = { - text: text, - **COMMON_DEFAULT_PARAMETERS, - } + private attr_accessor :token_keeper - parse_suggestions( - faraday.get(IdentityConfig.store.arcgis_api_suggest_url, params, dynamic_headers) do |req| - req.options.context = { service_name: 'arcgis_geocoder_suggest' } - end.body, - ) + def initialize(token_keeper: TokenKeeper.new) + @token_keeper = token_keeper end # Makes HTTP request to find a full address record using a magic key or single text line @@ -83,68 +67,34 @@ def find_address_candidates(**options) } parse_address_candidates( - faraday.get( - IdentityConfig.store.arcgis_api_find_address_candidates_url, params, - dynamic_headers + connection.get( + IdentityConfig.store.arcgis_api_find_address_candidates_url, params, dynamic_headers ) do |req| req.options.context = { service_name: 'arcgis_geocoder_find_address_candidates' } end.body, ) end - # Makes a request to retrieve a new token - # it expires after 1 hour - # @return [String] Auth token - def retrieve_token! - token, expires = request_token.fetch_values('token', 'expires') - expires_at = Time.zone.at(expires / 1000) - Rails.cache.write(API_TOKEN_CACHE_KEY, token, expires_at: expires_at) - # If using a redis cache we have to manually set the expires_at. This is because we aren't - # using a dedicated Redis cache and instead are just using our existing Redis server with - # mixed usage patterns. Without this cache entries don't expire. - # More at https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html - Rails.cache.try(:redis)&.expireat(API_TOKEN_CACHE_KEY, expires_at.to_i) - token - end - - # Checks the cache for an unexpired token and returns that. - # If the cache has expired, retrieves a new token and returns it - # @return [String] Auth token def token - Rails.cache.read(API_TOKEN_CACHE_KEY) || retrieve_token! + token_keeper&.token end private - def faraday + def connection Faraday.new do |conn| # Log request metrics conn.request :instrumentation, name: 'request_metric.faraday' - conn.options.timeout = IdentityConfig.store.arcgis_api_request_timeout_seconds - # Raise an error subclassing Faraday::Error on 4xx, 5xx, and malformed responses # Note: The order of this matters for parsing the error response body. conn.response :raise_error - # Parse JSON responses conn.response :json, content_type: 'application/json' - yield conn if block_given? end end - def parse_suggestions(response_body) - handle_api_errors(response_body) - - response_body['suggestions'].map do |suggestion| - Suggestion.new( - text: suggestion['text'], - magic_key: suggestion['magicKey'], - ) - end - end - def parse_address_candidates(response_body) handle_api_errors(response_body) @@ -180,7 +130,6 @@ def handle_api_errors(response_body) response_status_code: error_code, api_status_code: error_code, ) - Rails.cache.delete(API_TOKEN_CACHE_KEY) # this might only be needed for local testing raise Faraday::ClientError.new( RuntimeError.new(error_message), { @@ -199,26 +148,6 @@ def dynamic_headers { 'Authorization' => "Bearer #{token}" } end - # Makes HTTP request to authentication endpoint and - # returns the token and when it expires (1 hour). - # @return [Hash] API response - def request_token - body = { - username: IdentityConfig.store.arcgis_api_username, - password: IdentityConfig.store.arcgis_api_password, - referer: IdentityConfig.store.domain_name, - f: 'json', - } - - faraday.post( - IdentityConfig.store.arcgis_api_generate_token_url, URI.encode_www_form(body) - ) do |req| - req.options.context = { service_name: 'arcgis_token' } - end.body.tap do |body| - handle_api_errors(body) - end - end - def analytics(user: AnonymousUser.new) Analytics.new(user: user, request: nil, session: {}, sp: nil) end diff --git a/app/services/arcgis_api/mock/geocoder.rb b/app/services/arcgis_api/mock/geocoder.rb index d27b5a6f610..5e630f9e965 100644 --- a/app/services/arcgis_api/mock/geocoder.rb +++ b/app/services/arcgis_api/mock/geocoder.rb @@ -1,13 +1,56 @@ module ArcgisApi module Mock + class TokenKeeper < ArcgisApi::TokenKeeper + def connection + stubs = Faraday::Adapter::Test::Stubs.new do |stub| + stub_generate_token(stub) + end + super do |con| + con.adapter :test, stubs + end + end + + private + + def stub_generate_token(stub) + stub.post(IdentityConfig.store.arcgis_api_generate_token_url) do |env| + [ + 200, + { 'Content-Type': 'application/json' }, + { + token: '1234', + expires: (Time.zone.now + 1.minute).to_f * 1000, + ssl: true, + }.to_json, + ] + end + end + end + class Geocoder < ArcgisApi::Geocoder - def faraday - super do |conn| - conn.adapter :test do |stub| - stub_generate_token(stub) - stub_suggestions(stub) - stub_address_candidates(stub) - end + # def faraday + # super do |conn| + # conn.adapter :test do |stub| + # stub_generate_token(stub) + # stub_suggestions(stub) + # stub_address_candidates(stub) + # end + # end + # end + + def initialize + token_keeper = TokenKeeper.new + super(token_keeper: token_keeper) + end + + def connection + stubs = Faraday::Adapter::Test::Stubs.new do |stub| + stub_generate_token(stub) + stub_suggestions(stub) + stub_address_candidates(stub) + end + super do |con| + con.adapter :test, stubs end end @@ -20,7 +63,7 @@ def stub_generate_token(stub) { 'Content-Type': 'application/json' }, { token: '1234', - expires: 1234, + expires: (Time.zone.now + 30.seconds).to_f * 1000, ssl: true, }.to_json, ] diff --git a/app/services/arcgis_api/response_validation.rb b/app/services/arcgis_api/response_validation.rb new file mode 100644 index 00000000000..c6615f1618e --- /dev/null +++ b/app/services/arcgis_api/response_validation.rb @@ -0,0 +1,26 @@ +module ArcgisApi + # Faraday middleware to raise exception when response status is 200 but with error in body + class ResponseValidation < Faraday::Middleware + def on_complete(env) + return unless env[:status] == 200 && env.body + body = env.body.is_a?(String) ? JSON.parse(env.body) : env.body + return unless body.fetch('error', false) + handle_api_errors(body) + end + + def handle_api_errors(response_body) + # response_body is in this format: + # {"error"=>{"code"=>400, "message"=>"", "details"=>[""]}} + error_code = response_body.dig('error', 'code') + error_message = response_body.dig('error', 'message') || "Received error code #{error_code}" + # log an error + raise ArcgisApi::InvalidResponseError.new( + Faraday::Error.new(error_message), + { + status: error_code, + body: { details: response_body.dig('error', 'details')&.join(', ') }, + }, + ) + end + end +end diff --git a/app/services/arcgis_api/token_keeper.rb b/app/services/arcgis_api/token_keeper.rb new file mode 100644 index 00000000000..d1e651bc17e --- /dev/null +++ b/app/services/arcgis_api/token_keeper.rb @@ -0,0 +1,315 @@ +module ArcgisApi + # Struct to store token information, this allows us to track + # real expiration time with various rails cache backends many of them + # do not support entry expiration. + # Attributes + # token: the token string + # expires_at: hard expiration timestamp in epoch seconds + # sliding_expires_at: optional the token keeper to maintain for + # sliding expiration when sliding expiration enabled. + # A time that the token does not actually expire + # but used to control the timing of requesting a new token before it expires. + # It's initially set to expires_at - 3*prefetch_ttl. + TokenInfo = Struct.new( + :token, + :expires_at, + :sliding_expires_at, + ) + + class TokenCache + API_TOKEN_HOST = URI(IdentityConfig.store.arcgis_api_generate_token_url).host + API_TOKEN_CACHE_KEY = + "#{IdentityConfig.store.arcgis_api_token_cache_key_prefix}:#{API_TOKEN_HOST}" + + attr_accessor :cache_key + + def initialize(cache_key: API_TOKEN_CACHE_KEY) + @cache_key = cache_key + end + + # @return [String|nil] auth token + def token + token_entry&.token + end + + # @return [ArcgisApi:TokenInfo|nil] fetch cache entry and wrap it necessary, if miss return nil + def token_entry + cache_entry = Rails.cache.read(cache_key) + # return what we get from cache if found + return nil unless cache_entry + cache_entry = wrap_raw_token(cache_entry) unless cache_entry.is_a?(ArcgisApi::TokenInfo) + cache_entry + end + + # @param [ArcgisApi::TokenInfo] cache_value the value to write to cache + # @param [Number] expires_at the hard expiration time in unix time + def save_token(cache_value, expires_at) + entry_to_write = IdentityConfig.store.arcgis_token_sliding_expiration_enabled ? + cache_value : cache_value.token + Rails.cache.write(cache_key, entry_to_write, expires_at: expires_at) + # If using a redis cache we have to manually set the expires_at. This is because we aren't + # using a dedicated Redis cache and instead are just using our existing Redis server with + # mixed usage patterns. Without this cache entries don't expire. + # More at https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html + Rails.cache.try(:redis)&.expireat(cache_key, expires_at.to_i) + end + + def remove_token + Rails.cache.delete(cache_key) + end + + def wrap_raw_token(token, expires_at = nil) + ArcgisApi::TokenInfo.new( + token: token, + expires_at: expires_at, + ) + end + end + + # Class for retrieve, refresh and manage caching of Arcgis API token. + # If token synchronous fetching is disabled, a token will be fetched from + # cache directly or it will be a cache miss. + # Otherwise the thread will try to fetch/refresh the token on demand as needed. + # + # When sliding_expires_at<= current time <= sliding_expires_at + prefetch_ttl, + # the entry's sliding_expires_at time is updated to + # sliding_expires_at + prefetch_ttl and a new token is requested and saved to cache. + # + # When sliding_expires_at + prefetch_ttl < current time, + # a new token is requested and saved to cache. + # + # Optimistically, the token in cache will NOT expire + # when accessed by multi-threaded/distributed clients, + # since there is about expires_at - 2*prefetch_ttl + # length of time to generate a new API token. + + class TokenKeeper < TokenCache + API_PREFETCH_TTL_SECONDS = IdentityConfig.store.arcgis_api_token_prefetch_ttl_seconds + + RETRY_HTTP_STATUS = [404, 408, 409, 421, 429, 500, 502, 503, 504, 509] + + TIMES_TTL = 3 + + attr_accessor :prefetch_ttl, :analytics, :sliding_expiration_enabled, + :expiration_strategy + + # @param [String] cache_key token cache key + # @param [Number] prefetch_ttl number of seconds used to calculate a sliding_expires_at time + def initialize(cache_key: API_TOKEN_CACHE_KEY, + prefetch_ttl: API_PREFETCH_TTL_SECONDS) + super(cache_key: cache_key) + @prefetch_ttl = (prefetch_ttl && prefetch_ttl > 0 ? prefetch_ttl : API_PREFETCH_TTL_SECONDS) + @prefetch_ttl += (rand - 0.5) # random jitter + @analytics = Analytics.new(user: AnonymousUser.new, request: nil, session: {}, sp: nil) + @sliding_expiration_enabled = + IdentityConfig.store.arcgis_token_sliding_expiration_enabled + @expiration_strategy = + TokenExpirationStrategy.new(sliding_expiration_enabled: sliding_expiration_enabled) + end + + # Checks the cache for an unexpired token and returns that. + # If the cache has expired, retrieves a new token and returns it + # @return [ArcgisApi::TokenInfo] Auth token + def token_entry + cache_value = super + return cache_value unless IdentityConfig.store.arcgis_token_sync_request_enabled + # go get new token if needed and sync request enabled + if cache_value.nil? + cache_entry = retrieve_token + expires_at = cache_entry.expires_at + if sliding_expiration_enabled + cache_entry.sliding_expires_at = prefetch_ttl >= 0 ? + expires_at - TIMES_TTL * prefetch_ttl : expires_at + end + save_token(cache_entry, expires_at) + cache_entry + else + process_expired_token(cache_value) + end + end + + # Makes a request to retrieve a new token + # it expires after 1 hour + # @return [ArcgisApi::TokenInfo] Auth token + def retrieve_token + token, expires = request_token.fetch_values('token', 'expires') + expires_at = Time.zone.at(expires / 1000).to_f + return ArcgisApi::TokenInfo.new(token: token, expires_at: expires_at) + end + + private + + # Makes HTTP request to authentication endpoint and + # returns the token and when it expires (1 hour). + # @return [Hash] API response + def request_token + body = { + username: IdentityConfig.store.arcgis_api_username, + password: IdentityConfig.store.arcgis_api_password, + referer: IdentityConfig.store.domain_name, + f: 'json', + } + + connection.post( + IdentityConfig.store.arcgis_api_generate_token_url, URI.encode_www_form(body) + ) do |req| + req.options.context = { service_name: 'arcgis_token' } + end.body + end + + def connection + faraday_retry_options = { + retry_statuses: RETRY_HTTP_STATUS, + max: IdentityConfig.store.arcgis_get_token_max_retries, + methods: %i[post], + interval: IdentityConfig.store.arcgis_get_token_retry_interval_seconds, + interval_randomness: 0.25, + backoff_factor: IdentityConfig.store.arcgis_get_token_retry_backoff_factor, + exceptions: [Errno::ETIMEDOUT, Timeout::Error, Faraday::TimeoutError, Faraday::ServerError, + Faraday::ClientError, Faraday::RetriableResponse, + ArcgisApi::InvalidResponseError], + retry_block: ->(env:, options:, retry_count:, exception:, will_retry_in:) { + # log analytics event + exception_message = exception_message(exception, options, retry_count, will_retry_in) + notify_retry(env, exception_message) + }, + } + Faraday.new do |conn| + # Log request metrics + conn.request :instrumentation, name: 'request_metric.faraday' + conn.options.timeout = IdentityConfig.store.arcgis_api_request_timeout_seconds + # Parse JSON responses + conn.response :json, content_type: 'application/json' + # Raise an error subclassing Faraday::Error on 4xx, 5xx, and malformed responses + # Note: The order of this matters for parsing the error response body. + conn.response :raise_error + conn.request :retry, faraday_retry_options + conn.use ArcgisApi::ResponseValidation + yield conn if block_given? + end + end + + # Core logic on what to do when a token sliding-expired (very close to expiration time). + # Ideally this wont be invoked when there is a cache miss(already hard expired). + # Sliding expiration gives the application a buffer to preemptively get new token and + # avoid affecting large number of workers near the real expiration time. + # When it's sliding expired. the current worker first extend the sliding_expires_at + # time with an additional prefetch_ttl seconds (to mitigate chances other workers + # doing the same thing, not bullet proof, unless we have a single locking mechanism), + # then go ahead request a new token from arcgis and save it to cache. + # + # @param [ArcgisApi::TokenInfo] cache_value existing cache_entry, non nil value + # @return [ArcgisApi::TokenInfo] retrieve and save a new token if expired, + # or extend sliding_expires_at if needed + def process_expired_token(cache_value) + return cache_value unless expiration_strategy.expired?( + token_info: cache_value, + ) + # process sliding expired cache_value + # extend the sliding_expires_at with additional prefetch_ttl seconds value if needed + current_sliding_expires_at = cache_value.sliding_expires_at + expiration_strategy. + extend_sliding_expires_at(token_info: cache_value, prefetch_ttl: prefetch_ttl) + # avoid extra call if needed when nothing changes + if current_sliding_expires_at != cache_value.sliding_expires_at + save_token( + cache_value, + cache_value.expires_at, + ) + end + + # now retrieve new token + update_value = retrieve_token + expires_at = update_value&.expires_at + if sliding_expiration_enabled + update_value.sliding_expires_at = prefetch_ttl >= 0 ? + expires_at - TIMES_TTL * prefetch_ttl : expires_at + end + save_token( + update_value, + update_value.expires_at, + ) + return update_value + end + + def notify_retry(env, exception_message) + body = env.body + case body + when Hash + resp_body = body + when String + resp_body = begin + JSON.parse(body) + rescue + body + end + else + resp_body = body + end + http_status = env.status + api_status_code = resp_body.is_a?(Hash) ? resp_body.dig('error', 'code') : http_status + analytics.idv_arcgis_token_failure( + exception_class: 'ArcGIS', + exception_message: exception_message, + response_body_present: resp_body.present?, + response_body: resp_body, + response_status_code: http_status, + api_status_code: api_status_code, + ) + end + + def exception_message(exception, options, retry_count, will_retry_in) + # rubocop:disable Layout/LineLength + if options.max == retry_count + 1 + exception_message = "token request max retries(#{options.max}) reached, error : #{exception.message}" + else + exception_message = + "token request retry count : #{retry_count}, will retry in #{will_retry_in}, error : #{exception.message}" + end + # rubocop:enable Layout/LineLength + exception_message + end + end + + class TokenExpirationStrategy + attr_accessor :sliding_expiration_enabled + + def initialize(sliding_expiration_enabled:) + @sliding_expiration_enabled = sliding_expiration_enabled + end + + # Check whether a token_info expired or not. It's considered not expired + # when has no expires_at field. + # If expires_at time is past, then it's expired. Otherwise, check sliding_expires_at when + # sliding expiration is enabled, and it's considered expired when sliding_expires_at time + # is over prefetch_ttl seconds. + # + # @param [ArcgisApi::TokenInfo] token_info + # @return [true|false] whether it's considered expired + def expired?(token_info:) + return true unless token_info + expires_at = token_info&.expires_at + return false unless expires_at + now = Time.zone.now.to_f + # hard expired + return true if expires_at && expires_at <= now + # sliding expired + if sliding_expiration_enabled + sliding_expires_at = token_info&.sliding_expires_at + return true if sliding_expires_at && now >= sliding_expires_at + end + return false + end + + # update sliding_expires_at if necessary and extend it prefetch_ttl seconds + def extend_sliding_expires_at(token_info:, prefetch_ttl:) + if sliding_expiration_enabled && token_info&.sliding_expires_at + token_info.sliding_expires_at += prefetch_ttl + end + token_info + end + end + + class InvalidResponseError < Faraday::Error + end +end diff --git a/config/application.yml.default b/config/application.yml.default index bf2b8bf7eb9..6d9f1a2ec36 100644 --- a/config/application.yml.default +++ b/config/application.yml.default @@ -49,6 +49,13 @@ arcgis_api_generate_token_url: 'https://gis.gsa.gov/portal/sharing/rest/generate arcgis_api_suggest_url: 'https://gis.gsa.gov/servernh/rest/services/GSA/USA/GeocodeServer/suggest' arcgis_api_find_address_candidates_url: 'https://gis.gsa.gov/servernh/rest/services/GSA/USA/GeocodeServer/findAddressCandidates' arcgis_api_request_timeout_seconds: 5 +arcgis_api_token_prefetch_ttl_seconds: 10 +arcgis_api_token_cache_key_prefix: arcgis_api_token +arcgis_get_token_max_retries: 5 +arcgis_get_token_retry_interval_seconds: 1 +arcgis_get_token_retry_backoff_factor: 2 +arcgis_token_sliding_expiration_enabled: true +arcgis_token_sync_request_enabled: false asset_host: '' async_wait_timeout_seconds: 60 async_stale_job_timeout_seconds: 300 @@ -518,6 +525,7 @@ production: test: aamva_private_key: 123abc aamva_public_key: 123abc + arcgis_token_sync_request_enabled: true acuant_assure_id_url: https://example.com acuant_facial_match_url: https://facial_match.example.com acuant_passlive_url: https://liveness.example.com diff --git a/config/initializers/job_configurations.rb b/config/initializers/job_configurations.rb index 50163c4f8b5..e0f1f298725 100644 --- a/config/initializers/job_configurations.rb +++ b/config/initializers/job_configurations.rb @@ -3,6 +3,7 @@ cron_24h = '0 0 * * *' gpo_cron_24h = '0 10 * * *' # 10am UTC is 5am EST/6am EDT cron_1w = '0 0 * * 0' +cron_55m = '0/55 * * * *' if defined?(Rails::Console) Rails.logger.info 'job_configurations: console detected, skipping schedule' @@ -169,6 +170,10 @@ cron: cron_24h, args: -> { [Time.zone.yesterday] }, }, + arcgis_token: { + class: 'ArcgisTokenJob', + cron: cron_55m, + }, } end # rubocop:enable Metrics/BlockLength diff --git a/lib/identity_config.rb b/lib/identity_config.rb index 9ba1fd654d8..f21981a1cb3 100644 --- a/lib/identity_config.rb +++ b/lib/identity_config.rb @@ -126,6 +126,13 @@ def self.build_store(config_map) config.add(:arcgis_api_suggest_url, type: :string) config.add(:arcgis_api_find_address_candidates_url, type: :string) config.add(:arcgis_api_request_timeout_seconds, type: :integer) + config.add(:arcgis_api_token_cache_key_prefix, type: :string) + config.add(:arcgis_api_token_prefetch_ttl_seconds, type: :integer) + config.add(:arcgis_get_token_max_retries, type: :integer) + config.add(:arcgis_get_token_retry_interval_seconds, type: :integer) + config.add(:arcgis_get_token_retry_backoff_factor, type: :integer) + config.add(:arcgis_token_sync_request_enabled, type: :boolean) + config.add(:arcgis_token_sliding_expiration_enabled, type: :boolean) config.add(:aws_http_retry_limit, type: :integer) config.add(:aws_http_retry_max_delay, type: :integer) config.add(:aws_http_timeout, type: :integer) diff --git a/spec/features/idv/in_person_spec.rb b/spec/features/idv/in_person_spec.rb index aca237bd1a2..80455d2ca00 100644 --- a/spec/features/idv/in_person_spec.rb +++ b/spec/features/idv/in_person_spec.rb @@ -10,6 +10,7 @@ allow(IdentityConfig.store).to receive(:in_person_proofing_enabled).and_return(true) allow(IdentityConfig.store).to receive(:in_person_capture_secondary_id_enabled). and_return(false) + allow(IdentityConfig.store).to receive(:arcgis_token_sync_request_enabled).and_return(true) end context 'ThreatMetrix review pending' do diff --git a/spec/jobs/arcgis_token_job_spec.rb b/spec/jobs/arcgis_token_job_spec.rb new file mode 100644 index 00000000000..f301f04dd50 --- /dev/null +++ b/spec/jobs/arcgis_token_job_spec.rb @@ -0,0 +1,23 @@ +require 'rails_helper' + +RSpec.describe ArcgisTokenJob, type: :job do + let(:token_keeper) { instance_spy(ArcgisApi::TokenKeeper) } + let(:job) { described_class.new(token_keeper: token_keeper) } + let(:analytics) { instance_spy(Analytics) } + describe 'arcgis token job' do + it 'fetches token successfully' do + allow(job).to receive(:analytics).and_return(analytics) + allow(job).to receive(:token_keeper).and_return(token_keeper) + allow(token_keeper).to receive(:retrieve_token).and_return(ArcgisApi::TokenInfo.new) + expect(job.perform).to eq(true) + expect(token_keeper).to have_received(:retrieve_token).once + expect(token_keeper).to have_received(:save_token).once + expect(analytics).to have_received( + :idv_arcgis_token_job_started, + ).once + expect(analytics).to have_received( + :idv_arcgis_token_job_completed, + ).once + end + end +end diff --git a/spec/services/arcgis_api/connection_factory_spec.rb b/spec/services/arcgis_api/connection_factory_spec.rb new file mode 100644 index 00000000000..3a4051d0097 --- /dev/null +++ b/spec/services/arcgis_api/connection_factory_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' +RSpec.describe ArcgisApi::ConnectionFactory do + let(:subject) { described_class.new } + + context 'Create new connection' do + it 'create connection successfully' do + test_message = 'This is a test' + test_response = <<-BODY + { + "message": "#{test_message}" + } + BODY + stub_request(:get, 'https://google.com/'). + with( + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'User-Agent' => 'Faraday v2.7.4', + }, + ). + to_return(status: 200, body: test_response, headers: { content_type: 'application/json' }) + + conn = subject.connection do |con| + expect(con).to be_instance_of(Faraday::Connection) + end + + res = conn.get('https://google.com') do |req| + req.options.context = { service_name: 'arcgis_geocoder_suggest' } + end + expect(res.body.fetch('message')).to eq(test_message) + end + end +end diff --git a/spec/services/arcgis_api/geocoder_spec.rb b/spec/services/arcgis_api/geocoder_spec.rb index 10982dc880e..be747ffa24c 100644 --- a/spec/services/arcgis_api/geocoder_spec.rb +++ b/spec/services/arcgis_api/geocoder_spec.rb @@ -2,44 +2,13 @@ RSpec.describe ArcgisApi::Geocoder do include ArcgisApiHelper - + let(:cache_key) { 'test_arcgis_geocoder_token' } let(:subject) { ArcgisApi::Geocoder.new } - describe '#suggest' do - before(:each) do - stub_generate_token_response - end - - it 'returns suggestions' do - stub_request_suggestions - - suggestions = subject.suggest('100 Main') - - expect(suggestions.first.magic_key).to be_present - expect(suggestions.first.text).to be_present - end - - it 'returns an error response body but with Status coded as 200' do - stub_request_suggestions_error - - expect { subject.suggest('100 Main') }.to raise_error do |error| - expect(error).to be_instance_of(Faraday::ClientError) - expect(error.message).to eq('Unable to complete operation.') - expect(error.response).to be_kind_of(Hash) - end - end - - it 'returns an error with Status coded as 4** in HTML' do - stub_request_suggestions_error_html - - expect { subject.suggest('100 Main') }.to raise_error( - an_instance_of(Faraday::BadRequestError), - ) - end - end - describe '#find_address_candidates' do before(:each) do + allow(IdentityConfig.store).to receive(:arcgis_api_token_cache_key_prefix). + and_return(cache_key) stub_generate_token_response end @@ -105,18 +74,17 @@ end end - describe '#retrieve_token!' do + describe '#token!' do it 'sets token and token_expires_at' do stub_generate_token_response - subject.retrieve_token! - - expect(subject.token).to be_present + token = subject.token + expect(token).to be_present end it 'attempts to generate a token with invalid credentials' do stub_invalid_token_credentials_response - expect { subject.suggest('100 Main') }.to raise_error do |error| + expect { subject.find_address_candidates(SingleLine: 'abc123') }.to raise_error do |error| expect(error.message).to eq('Unable to generate token.') end end @@ -124,7 +92,7 @@ it 'calls the token service with no response' do stub_token_service_unreachable_response - expect { subject.suggest('100 Main') }.to raise_error do |error| + expect { subject.find_address_candidates(SingleLine: 'abc123') }.to raise_error do |error| expect(error.message).to eq('Failed to open TCP connection') end end @@ -139,7 +107,7 @@ allow(IdentityConfig.store).to receive(:arcgis_api_password). and_return(password) - subject.retrieve_token! + subject.token expect(WebMock).to have_requested(:post, %r{/generateToken}). with( @@ -149,16 +117,16 @@ it 'reuses the cached token on subsequent requests' do stub_generate_token_response - stub_request_suggestions - stub_request_suggestions - stub_request_suggestions + stub_request_candidates_response + stub_request_candidates_response + stub_request_candidates_response - subject.suggest('1') - subject.suggest('100') - subject.suggest('100 Main') + subject.find_address_candidates(SingleLine: 'abc1') + subject.find_address_candidates(SingleLine: 'abc12') + subject.find_address_candidates(SingleLine: 'abc123') expect(WebMock).to have_requested(:post, %r{/generateToken}).once - expect(WebMock).to have_requested(:get, %r{/suggest}).times(3) + expect(WebMock).to have_requested(:get, %r{/findAddressCandidates}).times(3) end it 'implicitly refreshes the token when expired' do @@ -167,50 +135,32 @@ and_return(root_url) stub_generate_token_response(expires_at: 1.hour.from_now.to_i * 1000, token: 'token1') - stub_request_suggestions - subject.suggest('100 Main') + stub_request_candidates_response + subject.find_address_candidates(SingleLine: 'abc123') travel 2.hours stub_generate_token_response(token: 'token2') - stub_request_suggestions - subject.suggest('100 Main') + stub_request_candidates_response + subject.find_address_candidates(SingleLine: 'abc123') expect(WebMock).to have_requested(:post, %r{/generateToken}).twice - expect(WebMock).to have_requested(:get, %r{/suggest}). + expect(WebMock).to have_requested(:get, %r{/findAddressCandidates}). with(headers: { 'Authorization' => 'Bearer token1' }).once - expect(WebMock).to have_requested(:get, %r{/suggest}). + expect(WebMock).to have_requested(:get, %r{/findAddressCandidates}). with(headers: { 'Authorization' => 'Bearer token1' }).once end it 'reuses the cached token across instances' do stub_generate_token_response(token: 'token1') - stub_request_suggestions - stub_request_suggestions - - client1 = ArcgisApi::Geocoder.new - client2 = ArcgisApi::Geocoder.new + stub_request_candidates_response + stub_request_candidates_response - client1.suggest('1') - client2.suggest('100') + subject.find_address_candidates(SingleLine: 'abc12') + subject.find_address_candidates(SingleLine: 'abc123') - expect(WebMock).to have_requested(:get, %r{/suggest}). + expect(WebMock).to have_requested(:get, %r{/findAddressCandidates}). with(headers: { 'Authorization' => 'Bearer token1' }).twice end - - context 'when using redis as a backing store' do - before do |ex| - allow(Rails).to receive(:cache).and_return( - ActiveSupport::Cache::RedisCacheStore.new(url: IdentityConfig.store.redis_throttle_url), - ) - end - - it 'manually sets the expiration' do - stub_generate_token_response - subject.retrieve_token! - ttl = Rails.cache.redis.ttl(ArcgisApi::Geocoder::API_TOKEN_CACHE_KEY) - expect(ttl).to be > 0 - end - end end end diff --git a/spec/services/arcgis_api/token_keeper_spec.rb b/spec/services/arcgis_api/token_keeper_spec.rb new file mode 100644 index 00000000000..8875803d483 --- /dev/null +++ b/spec/services/arcgis_api/token_keeper_spec.rb @@ -0,0 +1,292 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ArcgisApi::TokenKeeper do + let(:prefetch_ttl) { 1 } + let(:analytics) { instance_spy(Analytics) } + let(:cache_key) { 'test_arcgis_api_token' } + let(:subject) do + obj = described_class.new( + cache_key: cache_key, + prefetch_ttl: prefetch_ttl, + ) + obj.analytics = (analytics) + obj + end + + let(:expected) { 'ABCDEFG' } + let(:expected_sec) { 'GFEDCBA' } + let(:expires_at) { (Time.zone.now + 15.seconds).to_f * 1000 } + let(:cache) { Rails.cache } + + before(:each) do + allow(Rails).to receive(:cache).and_return(cache_store) + subject.remove_token + end + + shared_examples 'acquire token test' do + context 'token not expired and not in prefetch timeframe' do + it 'get same token at second call' do + stub_request(:post, %r{/generateToken}).to_return( + { status: 200, + body: { + token: expected, + expires: (Time.zone.now + 15.seconds).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + { status: 200, + body: { + token: expected_sec, + expires: (Time.zone.now + 1.hour).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + ) + # verify configuration + expect(subject.sliding_expiration_enabled).to be(true) + expect(IdentityConfig.store.arcgis_token_sync_request_enabled).to be(true) + + expect(Rails.cache).to receive(:read).with(kind_of(String)). + and_call_original + freeze_time do + token = subject.token + expect(token).to eq(expected) + end + + travel 1.second do + expect(Rails.cache).to receive(:read).with(kind_of(String)). + and_call_original + token = subject.token + expect(token).to eq(expected) + end + end + end + + context 'token not expired and but in prefetch timeframe' do + before(:each) do + stub_request(:post, %r{/generateToken}).to_return( + { status: 200, + body: { + token: expected, + expires: (Time.zone.now + 15.seconds).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + { status: 200, + body: { + token: expected_sec, + expires: (Time.zone.now + 1.hour).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + ) + end + context 'get token at different timing' do + let(:prefetch_ttl) { 3 } + it 'get same token between sliding_expires_at passed and sliding_expires_at+prefetch_ttl' do + expect(Rails.cache).to receive(:read).with(kind_of(String)). + and_call_original + freeze_time do + token = subject.token + expect(token).to eq(expected) + end + Rails.logger.debug { "#####now0=#{Time.zone.now.to_f}" } + + travel 1.second do + expect(Rails.cache).to receive(:read).with(kind_of(String)). + and_call_original + Rails.logger.debug { "#####now=#{Time.zone.now.to_f}" } + token = subject.token + Rails.logger.debug { "token sec : #{token}" } + expect(token).to eq(expected) + end + end + it 'regenerates token when passed sliding_expires_at+prefetch_ttl' do + expect(Rails.cache).to receive(:read).with(kind_of(String)). + and_call_original + + token = subject.token + expect(token).to eq(expected) + + travel 11.seconds do + expect(Rails.cache).to receive(:read).with(kind_of(String)). + and_call_original + token = subject.token + expect(token).to eq(expected_sec) + end + end + end + end + + context 'value only token in cache' do + before(:each) do + stub_request(:post, %r{/generateToken}).to_return( + { status: 200, + body: { + token: expected, + expires: (Time.zone.now + 15.seconds).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + ) + subject.save_token(expected, expires_at) + end + let(:prefetch_ttl) { 5 } + it 'should use deal with the value only token' do + token = subject.token + expect(token).to eq(expected) + end + end + end + context 'with in memory store' do + let(:cache_store) { ActiveSupport::Cache.lookup_store(:memory_store) } + context 'sliding expiration enabled' do + before(:each) do + allow(IdentityConfig.store).to receive(:arcgis_token_sliding_expiration_enabled). + and_return(true) + end + include_examples 'acquire token test' + end + end + context 'with redis store' do + let(:cache_store) do + ActiveSupport::Cache.lookup_store(:redis_cache_store, { url: IdentityConfig.store.redis_url }) + end + include_examples 'acquire token test' + context 'retry options' do + it 'retry remote request multiple times as needed and emit analytics events' do + allow(IdentityConfig.store).to receive(:arcgis_get_token_max_retries).and_return(5) + stub_request(:post, %r{/generateToken}).to_return( + { + status: 503, + }, + { + status: 200, + body: ArcgisApi::Mock::Fixtures.request_token_service_error, + headers: { content_type: 'application/json;charset=UTF-8' }, + }, + { + status: 200, + body: ArcgisApi::Mock::Fixtures.invalid_gis_token_credentials_response, + headers: { content_type: 'application/json;charset=UTF-8' }, + }, + { status: 200, + body: { + token: expected, + expires: (Time.zone.now + 1.hour).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + ) + token = subject.retrieve_token + expect(token&.token).to eq(expected) + expect(analytics).to have_received(:idv_arcgis_token_failure).exactly(3).times + end + + it 'raises exception after max retries and log event correctly' do + allow(IdentityConfig.store).to receive(:arcgis_get_token_max_retries).and_return(2) + stub_request(:post, %r{/generateToken}).to_return( + { + status: 503, + }, + { + status: 429, + }, + { + status: 504, + }, + ) + expect do + subject.retrieve_token + end.to raise_error(Faraday::Error) + + msgs = [] + expect(analytics).to have_received(:idv_arcgis_token_failure) { |method_args| + msg = method_args.fetch(:exception_message) + msgs << msg + }.exactly(2).times.ordered + expect(msgs[0]).to match(/retry count/) + expect(msgs[1]).to match(/max retries/) + end + end + context 'token sync request disabled' do + it 'does not fetch token' do + allow(IdentityConfig.store).to receive(:arcgis_token_sync_request_enabled). + and_return(false) + expect(subject.token).to be(nil) + end + end + context 'sync request enabled and sliding expiration disabled' do + let(:original_token) { ArcgisApi::TokenInfo.new('12345', (Time.zone.now + 15.seconds).to_f) } + before(:each) do + allow(IdentityConfig.store).to receive(:arcgis_token_sync_request_enabled).and_return(true) + allow(IdentityConfig.store).to receive(:arcgis_token_sliding_expiration_enabled). + and_return(false) + stub_request(:post, %r{/generateToken}).to_return( + { status: 200, + body: { + token: expected, + expires: (Time.zone.now + 15.seconds).to_f * 1000, + ssl: true, + }.to_json, + headers: { content_type: 'application/json;charset=UTF-8' } }, + ) + # this will save raw token + subject.save_token(original_token, expires_at) + end + let(:prefetch_ttl) { 5 } + it 'should get token after existing one expired' do + freeze_time do + token = subject.token + expect(token).to eq(original_token.token) + end + travel 20.seconds do + # now simulate a cache miss, redis not affected by time travel + # Even with memory store, the original cache entry is a raw string + # rails cache won't expires it since no expiration information is available. + subject.remove_token + token = subject.token + expect(token).to eq(expected) + end + end + end + end +end + +RSpec.describe ArcgisApi::TokenExpirationStrategy do + describe 'when sliding_expiration_enabled is true' do + let(:subject) { ArcgisApi::TokenExpirationStrategy.new(sliding_expiration_enabled: true) } + it 'checks token expiration when now passed expires_at' do + expect(subject.expired?(token_info: nil)).to be(true) + token_info = ArcgisApi::TokenInfo.new(token: 'ABCDE') + # missing expires_at assume the token is valid + expect(subject.expired?(token_info: token_info)).to be(false) + freeze_time do + now = Time.zone.now.to_f + token_info.expires_at = now + expect(subject.expired?(token_info: token_info)).to be(true) + token_info.expires_at = now + 0.0001 + expect(subject.expired?(token_info: token_info)).to be(false) + end + end + context 'when sliding_expires_at <= now' do + it 'should expired and sliding_expires_at extended' do + freeze_time do + now = Time.zone.now.to_f + prefetch_ttl = 3 + token_info = ArcgisApi::TokenInfo.new( + token: 'ABCDE', + expires_at: now + 2 * prefetch_ttl, + sliding_expires_at: now - prefetch_ttl, + ) + expect(subject.expired?(token_info: token_info)).to be(true) + existing_sliding_expires = token_info.sliding_expires_at + subject.extend_sliding_expires_at(token_info: token_info, prefetch_ttl: prefetch_ttl) + expect(token_info.sliding_expires_at - existing_sliding_expires).to eq(prefetch_ttl) + end + end + end + end +end diff --git a/spec/support/arcgis_api_helper.rb b/spec/support/arcgis_api_helper.rb index 5ef1b3cf427..2c38e622b79 100644 --- a/spec/support/arcgis_api_helper.rb +++ b/spec/support/arcgis_api_helper.rb @@ -1,24 +1,4 @@ module ArcgisApiHelper - def stub_request_suggestions - stub_request(:get, %r{/suggest}).to_return( - status: 200, body: ArcgisApi::Mock::Fixtures.request_suggestions_response, - headers: { content_type: 'application/json;charset=UTF-8' } - ) - end - - def stub_request_suggestions_error - stub_request(:get, %r{/suggest}).to_return( - status: 200, body: ArcgisApi::Mock::Fixtures.request_suggestions_error, - headers: { content_type: 'application/json;charset=UTF-8' } - ) - end - - def stub_request_suggestions_error_html - stub_request(:get, %r{/suggest}).to_return( - status: 400, body: ArcgisApi::Mock::Fixtures.request_suggestions_error_html, - ) - end - def stub_request_candidates_response stub_request(:get, %r{/findAddressCandidates}).to_return( status: 200, body: ArcgisApi::Mock::Fixtures.request_candidates_response,