added sleep time between paging ListResourceRecordSet

charts/external-dns/templates/deployment.yaml

@@ -118,6 +118,9 @@ spec:
           {{- range .Values.extraArgs }}
             - {{ tpl . $ }}
           {{- end }}
+          {{- if .Values.awsPagingInterval }}
+            - --aws-paging-interval={{ .Values.awsPagingInterval }}
+          {{- end }}
           ports:
             - name: http
               protocol: TCP

docs/tutorials/aws.md

@@ -994,6 +994,8 @@ Running several fast polling ExternalDNS instances in a given account can easily
   * `--aws-batch-change-size=4000` (default `1000`)
 * Increase the interval between changes
   * `--aws-batch-change-interval=10s` (default `1s`)
+* Increase the interval between paging through records (ListResourceRecordSet)
+  * `--aws-paging-interval=500` in milliseconds (default 0)
 * Introducing some jitter to the pod initialization, so that when multiple instances of ExternalDNS are updated at the same time they do not make their requests on the same second.
 
 A simple way to implement randomised startup is with an init container:

main.go

@@ -232,6 +232,7 @@ func main() {
 				PreferCNAME:           cfg.AWSPreferCNAME,
 				DryRun:                cfg.DryRun,
 				ZoneCacheDuration:     cfg.AWSZoneCacheDuration,
+				PagingInterval: 			 cfg.AWSPagingInterval,
 			},
 			clients,
 		)

pkg/apis/externaldns/types.go

@@ -98,6 +98,7 @@ type Config struct {
 	AWSZoneMatchParent                 bool
 	AWSDynamoDBRegion                  string
 	AWSDynamoDBTable                   string
+	AWSPagingInterval                  time.Duration
 	AzureConfigFile                    string
 	AzureResourceGroup                 string
 	AzureSubscriptionID                string
@@ -267,6 +268,7 @@ var defaultConfig = &Config{
 	AWSSDServiceCleanup:         false,
 	AWSDynamoDBRegion:           "",
 	AWSDynamoDBTable:            "external-dns",
+	AWSPagingInterval            0,
 	AzureConfigFile:             "/etc/kubernetes/azure.json",
 	AzureResourceGroup:          "",
 	AzureSubscriptionID:         "",
@@ -477,6 +479,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("aws-batch-change-size-bytes", "When using the AWS provider, set the maximum byte size that will be applied in each batch.").Default(strconv.Itoa(defaultConfig.AWSBatchChangeSizeBytes)).IntVar(&cfg.AWSBatchChangeSizeBytes)
 	app.Flag("aws-batch-change-size-values", "When using the AWS provider, set the maximum total record values that will be applied in each batch.").Default(strconv.Itoa(defaultConfig.AWSBatchChangeSizeValues)).IntVar(&cfg.AWSBatchChangeSizeValues)
 	app.Flag("aws-batch-change-interval", "When using the AWS provider, set the interval between batch changes.").Default(defaultConfig.AWSBatchChangeInterval.String()).DurationVar(&cfg.AWSBatchChangeInterval)
+	app.Flag("aws-paging-interval", "When using the AWS provider, set the interval between paging throgugh ListResourceRecordSet calls.").Default(defaultConfig.AWSPagingInterval.String()).DurationVar(&cfg.AWSPagingInterval)
 	app.Flag("aws-evaluate-target-health", "When using the AWS provider, set whether to evaluate the health of a DNS target (default: enabled, disable with --no-aws-evaluate-target-health)").Default(strconv.FormatBool(defaultConfig.AWSEvaluateTargetHealth)).BoolVar(&cfg.AWSEvaluateTargetHealth)
 	app.Flag("aws-api-retries", "When using the AWS API, set the maximum number of retries before giving up.").Default(strconv.Itoa(defaultConfig.AWSAPIRetries)).IntVar(&cfg.AWSAPIRetries)
 	app.Flag("aws-prefer-cname", "When using the AWS provider, prefer using CNAME instead of ALIAS (default: disabled)").BoolVar(&cfg.AWSPreferCNAME)

provider/aws/aws.go

@@ -276,6 +276,7 @@ type AWSConfig struct {
 	PreferCNAME           bool
 	DryRun                bool
 	ZoneCacheDuration     time.Duration
+	PagingInterval        time.Duration
 }
 
 // NewAWSProvider initializes a new AWS Route53 based Provider.
@@ -295,6 +296,7 @@ func NewAWSProvider(awsConfig AWSConfig, clients map[string]Route53API) (*AWSPro
 		preferCNAME:           awsConfig.PreferCNAME,
 		dryRun:                awsConfig.DryRun,
 		zonesCache:            &zonesListCache{duration: awsConfig.ZoneCacheDuration},
+		pagingInterval         awsConfig.PagingInterval,
 		failedChangesQueue:    make(map[string]Route53Changes),
 	}
 
@@ -413,8 +415,15 @@ func (p *AWSProvider) Records(ctx context.Context) (endpoints []*endpoint.Endpoi
 	return p.records(ctx, zones)
 }
 
-func (p *AWSProvider) records(ctx context.Context, zones map[string]*profiledZone) ([]*endpoint.Endpoint, error) {
+func (p *AWSProvider) records(ctx context.Context, zones map[string]*profiledZone, pagingInterval time.Duration) ([]*endpoint.Endpoint, error) {
 	endpoints := make([]*endpoint.Endpoint, 0)
+	pagingInterval := 0 * time.Second // default paging value
+	if p.awsPagingInterval != "" {
+		interval, err := time.ParseDuration(p.awsPagingInterval)
+		if err == nil {
+			pagingInterval = interval
+		}
+	}
 	f := func(resp *route53.ListResourceRecordSetsOutput, lastPage bool) (shouldContinue bool) {
 		for _, r := range resp.ResourceRecordSets {
 			newEndpoints := make([]*endpoint.Endpoint, 0)
@@ -488,7 +497,7 @@ func (p *AWSProvider) records(ctx context.Context, zones map[string]*profiledZon
 				endpoints = append(endpoints, ep)
 			}
 		}
-
+		time.Sleep(pagingInterval) // Added sleep interval between paging
 		return true
 	}