Add support for invalidating multiple CF distributions with --cf-inval

when the bucket is the origin for multiple distributions

S3/CloudFront.py

@@ -11,6 +11,7 @@
 import random
 from datetime import datetime
 from logging import debug, info, warning, error
+from sets import Set
 
 try:
     import xml.etree.ElementTree as ET
@@ -447,7 +448,7 @@ def InvalidateObjects(self, uri, paths, default_index_file, invalidate_default_i
             paths = new_paths
 
         # uri could be either cf:// or s3:// uri
-        cfuri = self.get_dist_name_for_bucket(uri)
+        cfuris = self.get_dist_name_for_bucket(uri)
         if len(paths) > 999:
             try:
                 tmp_filename = Utils.mktmpfile()
@@ -459,16 +460,22 @@ def InvalidateObjects(self, uri, paths, default_index_file, invalidate_default_i
             except:
                 pass
             raise ParameterError("Too many paths to invalidate")
-        invalbatch = InvalidationBatch(distribution = cfuri.dist_id(), paths = paths)
-        debug("InvalidateObjects(): request_body: %s" % invalbatch)
-        response = self.send_request("Invalidate", dist_id = cfuri.dist_id(),
-                                     body = str(invalbatch))
-        response['dist_id'] = cfuri.dist_id()
-        if response['status'] == 201:
-            inval_info = Invalidation(response['data']).info
-            response['request_id'] = inval_info['Id']
-        debug("InvalidateObjects(): response: %s" % response)
-        return response
+
+        responses = []
+        for cfuri in cfuris:
+            invalbatch = InvalidationBatch(distribution = cfuri.dist_id(), paths = paths)
+            debug("InvalidateObjects(): request_body: %s" % invalbatch)
+            response = self.send_request("Invalidate", dist_id = cfuri.dist_id(),
+                                         body = str(invalbatch))
+            response['dist_id'] = cfuri.dist_id()
+            if response['status'] == 201:
+                inval_info = Invalidation(response['data']).info
+                response['request_id'] = inval_info['Id']
+            debug("InvalidateObjects(): response: %s" % response)
+
+            responses.append(response)
+
+        return responses
 
     def GetInvalList(self, cfuri):
         if cfuri.type != "cf":
@@ -578,19 +585,26 @@ def get_dist_name_for_bucket(self, uri):
             response = self.GetList()
             CloudFront.dist_list = {}
             for d in response['dist_list'].dist_summs:
+                distListIndex = ""
+
                 if d.info.has_key("S3Origin"):
-                    CloudFront.dist_list[getBucketFromHostname(d.info['S3Origin']['DNSName'])[0]] = d.uri()
+                    distListIndex = getBucketFromHostname(d.info['S3Origin']['DNSName'])[0]
                 elif d.info.has_key("CustomOrigin"):
                     # Aral: This used to skip over distributions with CustomOrigin, however, we mustn't
                     #       do this since S3 buckets that are set up as websites use custom origins.
                     #       Thankfully, the custom origin URLs they use start with the URL of the
                     #       S3 bucket. Here, we make use this naming convention to support this use case.
                     distListIndex = getBucketFromHostname(d.info['CustomOrigin']['DNSName'])[0];
                     distListIndex = distListIndex[:len(uri.bucket())]
-                    CloudFront.dist_list[distListIndex] = d.uri()
                 else:
                     # Aral: I'm not sure when this condition will be reached, but keeping it in there.
                     continue
+
+                if CloudFront.dist_list.get(distListIndex, None) is None:
+                    CloudFront.dist_list[distListIndex] = Set()
+
+                CloudFront.dist_list[distListIndex].add(d.uri())
+
             debug("dist_list: %s" % CloudFront.dist_list)
         try:
             return CloudFront.dist_list[uri.bucket()]
@@ -624,8 +638,8 @@ def _parse_args(args):
         cf = CloudFront(Config())
         cfuris = []
         for arg in args:
-            uri = cf.get_dist_name_for_bucket(S3Uri(arg))
-            cfuris.append(uri)
+            uris = cf.get_dist_name_for_bucket(S3Uri(arg))
+            cfuris.extend(uris)
         return cfuris
 
     @staticmethod

s3cmd

@@ -1447,11 +1447,11 @@ def cmd_sync_local2remote(args):
               if len(default_index_file) < 1:
                   default_index_file = None
 
-        result = cf.InvalidateObjects(destination_base_uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf)
-        if result['status'] == 201:
-            output("Created invalidation request for %d paths" % len(uploaded_objects_list))
-            output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
-
+        results = cf.InvalidateObjects(destination_base_uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf)
+        for result in results:
+            if result['status'] == 201:
+                output(u"Created invalidation request for %d paths" % len(uploaded_objects_list))
+                output(u"Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
 
     # main execution
     s3 = S3(cfg)