handle rate limit and implement retry logic (#207)

* Add test

* handle rate limit and implement retry logic

* Expose response headers into instance variables

* Fix tests

* Add test for client options

* remove unused code

* Add test case

.rubocop.yml

@@ -124,6 +124,9 @@ Style/TernaryParentheses:
 Style/PercentLiteralDelimiters:
   Enabled: false
 
+Style/BracesAroundHashParameters:
+  Enabled: false
+
 Naming/VariableNumber:
   Enabled: false
 

lib/twitter-ads/client.rb

@@ -34,7 +34,7 @@ def initialize(consumer_key, consumer_secret, access_token, access_token_secret,
       @consumer_secret     = consumer_secret
       @access_token        = access_token
       @access_token_secret = access_token_secret
-      @options             = opts
+      @options             = opts.fetch(:options, {})
       validate
       self
     end

lib/twitter-ads/cursor.rb

@@ -112,6 +112,12 @@ def fetch_next
     def from_response(response)
       @next_cursor = response.body[:next_cursor]
       @total_count = response.body[:total_count].to_i if response.body.key?(:total_count)
+
+      TwitterAds::Utils.extract_response_headers(response.headers).each { |key, value|
+        singleton_class.class_eval { attr_accessor key }
+        instance_variable_set("@#{key}", value)
+      }
+
       response.body.fetch(:data, []).each do |object|
         @collection << if @klass&.method_defined?(:from_response)
                          @klass.new(

lib/twitter-ads/error.rb

@@ -59,18 +59,7 @@ def from_response(object)
 
   # Server Errors (5XX)
   class ServerError < Error; end
-
-  class ServiceUnavailable < ServerError
-    attr_reader :retry_after
-
-    def initialize(object)
-      super object
-      if object.headers['retry-after']
-        @retry_after = object.headers['retry-after']
-      end
-      self
-    end
-  end
+  class ServiceUnavailable < ServerError; end
 
   # Client Errors (4XX)
   class ClientError < Error; end
@@ -80,12 +69,13 @@ class NotFound < ClientError; end
   class BadRequest < ClientError; end
 
   class RateLimit < ClientError
-    attr_reader :reset_at, :retry_after
+    attr_reader :reset_at
 
     def initialize(object)
       super object
-      @retry_after = object.headers['retry-after']
-      @reset_at = object.headers['rate_limit_reset']
+      header = object.headers.fetch('x-account-rate-limit-reset', nil) ||
+               object.headers.fetch('x-rate-limit-reset', nil)
+      @reset_at = header.first.to_i
       self
     end
   end

lib/twitter-ads/http/request.rb

@@ -71,8 +71,37 @@ def oauth_request
       token    = OAuth::AccessToken.new(consumer, @client.access_token, @client.access_token_secret)
       request.oauth!(consumer.http, consumer, token)
 
+      handle_rate_limit = @client.options.fetch(:handle_rate_limit, false)
+      retry_max         = @client.options.fetch(:retry_max, 0)
+      retry_delay       = @client.options.fetch(:retry_delay, 1500)
+      retry_on_status   = @client.options.fetch(:retry_on_status, [500, 503])
+      retry_count       = 0
+      retry_after       = nil
+
       write_log(request) if @client.options[:trace]
-      response = consumer.http.request(request)
+      while retry_count <= retry_max
+        response = consumer.http.request(request)
+        status_code = response.code.to_i
+        break if status_code >= 200 && status_code < 300
+
+        if handle_rate_limit && retry_after.nil?
+          rate_limit_reset = response.fetch('x-account-rate-limit-reset', nil) ||
+                             response.fetch('x-rate-limit-reset', nil)
+          if status_code == 429
+            retry_after = rate_limit_reset.to_i - Time.now.to_i
+            @client.logger.warn('Request reached Rate Limit: resume in %d seconds' % retry_after)
+            sleep(retry_after + 5)
+            next
+          end
+        end
+
+        if retry_max.positive?
+          break unless retry_on_status.include?(status_code)
+          sleep(retry_delay / 1000)
+        end
+
+        retry_count += 1
+      end
       write_log(response) if @client.options[:trace]
 
       Response.new(response.code, response.each {}, response.body)

lib/twitter-ads/http/response.rb

@@ -9,9 +9,7 @@ class Response
     attr_reader :code,
                 :headers,
                 :raw_body,
-                :body,
-                :rate_limit_remaining,
-                :rate_limit_reset
+                :body
 
     # Creates a new Response object instance.
     #
@@ -37,16 +35,6 @@ def initialize(code, headers, body)
         @body = raw_body
       end
 
-      if headers.key?('x-rate-limit-reset')
-        @rate_limit           = headers['x-rate-limit-limit'].first
-        @rate_limit_remaining = headers['x-rate-limit-remaining'].first
-        @rate_limit_reset     = headers['x-rate-limit-reset'].first.to_i
-      elsif headers.key?('x-cost-rate-limit-reset')
-        @rate_limit           = headers['x-cost-rate-limit-limit'].first
-        @rate_limit_remaining = headers['x-cost-rate-limit-remaining'].first
-        @rate_limit_reset     = Time.at(headers['x-cost-rate-limit-reset'].first.to_i)
-      end
-
       self
     end
 

lib/twitter-ads/resources/dsl.rb

@@ -20,7 +20,14 @@ module InstanceMethods
       # @return [self] A fully hydrated instance of the current class.
       #
       # @since 0.1.0
-      def from_response(object)
+      def from_response(object, headers = nil)
+        if !headers.nil?
+          TwitterAds::Utils.extract_response_headers(headers).each { |key, value|
+            singleton_class.class_eval { attr_accessor key }
+            instance_variable_set("@#{key}", value)
+          }
+        end
+
         self.class.properties.each do |name, type|
           value = nil
           if type == :time && object[name] && !object[name].empty?

lib/twitter-ads/utils.rb

@@ -70,6 +70,18 @@ def deprecated(name, opts = {})
         warn message
       end
 
+      def extract_response_headers(headers)
+        values = {}
+        # only get "X-${name}" custom response headers
+        headers.each { |key, value|
+          if key =~ /^x-/
+            values[key.gsub(/^x-/, '').tr('-', '_')] = \
+              value.first =~ /^[0-9]*$/ ? value.first.to_i : value.first
+          end
+        }
+        values
+      end
+
     end
 
   end

spec/fixtures/tweet_previews.json

@@ -0,0 +1,23 @@
+{
+  "data_type": "tweet_previews",
+  "request": {
+    "params": {
+      "tweet_ids": [
+        "1130942781109596160",
+        "1101254234031370240"
+      ],
+      "tweet_type": "PUBLISHED",
+      "account_id": "2iqph"
+    }
+  },
+  "data": [
+    {
+      "tweet_id": "1130942781109596160",
+      "preview": "<iframe class='tweet-preview' src='https://ton.smf1.twitter.com/ads-manager/tweet-preview/index.html?data=c29tZSByYW5kb20gYmFzZTY0IHN0cmluZ3MgaGVyZS4uLg=='>"
+    },
+    {
+      "tweet_id": "1101254234031370240",
+      "preview": "<iframe class='tweet-preview' src='https://ton.smf1.twitter.com/ads-manager/tweet-preview/index.html?data=c29tZSByYW5kb20gYmFzZTY0IHN0cmluZ3MgaGVyZS4uLg=='>"
+    }
+  ]
+}

spec/twitter-ads/client_spec.rb

@@ -55,10 +55,26 @@
 
     it 'allows additional options' do
       expect {
-        Client.new(consumer_key, consumer_secret, access_token, {})
+        Client.new(consumer_key, consumer_secret, access_token, access_token_secret, options: {})
       }.not_to raise_error
     end
 
+    it 'test client options' do
+      client = Client.new(
+        consumer_key,
+        consumer_secret,
+        access_token,
+        access_token_secret,
+        options: {
+          handle_rate_limit: true,
+          retry_max: 1,
+          retry_delay: 3000,
+          retry_on_status: [404, 500, 503]
+        }
+      )
+      expect(client.options.length).to eq 4
+    end
+
   end
 
   describe '#inspect' do

spec/twitter-ads/creative/tweet_previews_spec.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+# Copyright (C) 2019 Twitter, Inc.
+
+require 'spec_helper'
+
+include TwitterAds::Enum
+
+describe TwitterAds::Creative::TweetPreview do
+
+  let!(:resource) { "#{ADS_API}/accounts/2iqph/tweet_previews" }
+
+  before(:each) do
+    stub_fixture(:get, :accounts_load, "#{ADS_API}/accounts/2iqph")
+    stub_fixture(:get, :tweet_previews, /#{resource}\?.*/)
+  end
+
+  let(:client) do
+    Client.new(
+      Faker::Lorem.characters(40),
+      Faker::Lorem.characters(40),
+      Faker::Lorem.characters(40),
+      Faker::Lorem.characters(40)
+    )
+  end
+
+  let(:account) { client.accounts('2iqph') }
+  let(:instance) { described_class.new(account) }
+
+  it 'inspect TweetPreview.load() response' do
+    preview = instance.load(
+      account,
+      tweet_ids: %w(1130942781109596160 1101254234031370240),
+      tweet_type: TweetType::PUBLISHED)
+
+    expect(preview).to be_instance_of(Cursor)
+    expect(preview.count).to eq 2
+    tweet = preview.first
+    expect(tweet.tweet_id).to eq '1130942781109596160'
+  end
+
+end