Merge pull request #243 from cumulus-nasa/CUMULUS-326

Cumulus 326

.circleci/config.yml

@@ -177,11 +177,11 @@ jobs:
       - run:
           name: Install Dependencies
           command: |
-            yarn install
+            yarn
 
             # short fix for deployment package missing dependencies during deploy
             cd packages/deployment
-            npm install
+            yarn 
             cd ~/project
 
       - run:

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - `yarn e2e` command is available for end to end testing
 ### Fixed
 
+- **CUMULUS-326: "Occasionally encounter "Too Many Requests" on deployment"** The api gateway calls will handle throttling errors
 - **CUMULUS-175: "Dashboard providers not in sync with AWS providers."** The root cause of this bug - DynamoDB operations not showing up in Elasticsearch - was shared by collections and rules. The fix was to update providers', collections' and rules; POST, PUT and DELETE endpoints to operate on DynamoDB and using DynamoDB streams to update Elasticsearch. The following packages were made:
   - `@cumulus/deployment` deploys DynamoDB streams for the Collections, Providers and Rules tables as well as a new lambda function called `dbIndexer`. The `dbIndexer` lambda has an event source mapping which listens to each of the DynamoDB streams. The dbIndexer lambda receives events referencing operations on the DynamoDB table and updates the elasticsearch cluster accordingly.
   - The `@cumulus/api` endpoints for collections, providers and rules _only_ query DynamoDB, with the exception of LIST endpoints and the collections' GET endpoint.

README.md

@@ -99,7 +99,7 @@ Lerna handles the process of deciding which version number should be used as lon
 
 To update cumulus' version number run:
 
-     $ npm run update
+     $ yarn update
 
 You will be prompted to select the type of change (patch/minor/major). 
 
@@ -121,4 +121,4 @@ All packages on master branch are automatically published to NPM.
 
 ## Cleaning Up all the repos
 
-    $ npm run clean
+    $ yarn clean

packages/deployment/lib/kes.js

@@ -31,6 +31,14 @@ const { fetchMessageAdapter } = require('./adapter');
 const { extractCumulusConfigFromSF, generateTemplates } = require('./message');
 
 
+/**
+ * Makes setTimeout return a promise
+ *
+ * @param {integer} ms - number of milliseconds
+ * @returns {Promise} the arguments passed after the timeout
+ */
+const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+
 /**
  * A subclass of Kes class that overrides opsStack method.
  * The subclass checks whether the public/private keys are generated
@@ -66,10 +74,24 @@ class UpdatedKes extends Kes {
    * @returns {Promise.<boolean>} returns true if successful
    */
   async redeployApiGateWay(name, restApiId, stageName) {
+    const waitTime = 20;
     if (restApiId) {
-      const apigateway = new this.AWS.APIGateway();
-      await apigateway.createDeployment({ restApiId, stageName }).promise();
-      console.log(`${name} endpoints with the id ${restApiId} redeployed.`);
+      try {
+        const apigateway = new this.AWS.APIGateway();
+        await apigateway.createDeployment({ restApiId, stageName }).promise();
+        console.log(`${name} endpoints with the id ${restApiId} redeployed.`);
+      }
+      catch (e) {
+        if (e.message && e.message.includes('Too Many Requests')) {
+          console.log(
+            `Redeploying ${restApiId} was throttled. ` +
+            `Another attempt will be made in ${waitTime} seconds`
+          );
+          await delay(waitTime * 1000);
+          return this.redeployApiGateWay(name, restApiId, stageName);
+        }
+        throw e;
+      }
     }
     return true;
   }
@@ -193,13 +215,11 @@ class UpdatedKes extends Kes {
         return generateTemplates(this.config, outputs, this.uploadToS3.bind(this));
       })
       .then(() => this.restartECSTasks(this.config))
-      .then(() => {
-        const updates = [
-          this.redeployApiGateWay('api', apis.api, apis.stageName),
-          this.redeployApiGateWay('distribution', apis.distribution, apis.stageName)
-        ];
-
-        return Promise.all(updates);
+      .then(() => this.redeployApiGateWay('api', apis.api, apis.stageName))
+      .then(() => this.redeployApiGateWay('distribution', apis.distribution, apis.stageName))
+      .catch((e) => {
+        console.log(e);
+        throw e;
       });
   }
 }