BackupStatusMgr (#2) (#570)

1. Make backup/snapshot status manager store additional details like meta file location.
2. Ensure it can save last 60 snapshot days.
3. Allow bindable values on how to save snapshot status in file/datastore. Default implementation stores in a file on the local instance.
4. Non-backward compatible. The previous backup file gets overwritten with new values(null) during the upgrade process.

Author: arunagrawal-84

gradle/wrapper/gradle-wrapper.jar

@@ -1,4 +1,4 @@
-#Wed Jun 28 17:25:48 PDT 2017
+#Thu Jun 29 13:18:34 PDT 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

gradle/wrapper/gradle-wrapper.properties

@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -155,7 +155,7 @@ if $cygwin ; then
 fi
 
 # Escape application args
-save ( ) {
+save () {
     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
     echo " "
 }

gradlew

@@ -15,7 +15,9 @@
  */
 package com.netflix.priam;
 
+import com.amazonaws.services.simpledb.AmazonSimpleDB;
 import com.amazonaws.services.simpledb.AmazonSimpleDBClient;
+import com.amazonaws.services.simpledb.AmazonSimpleDBClientBuilder;
 import com.amazonaws.services.simpledb.model.Attribute;
 import com.amazonaws.services.simpledb.model.Item;
 import com.amazonaws.services.simpledb.model.SelectRequest;
@@ -65,7 +67,7 @@ public void intialize(final String asgName, final String region)
         super.intialize(asgName, region);
 
         // End point is us-east-1
-        AmazonSimpleDBClient simpleDBClient = new AmazonSimpleDBClient(provider.getAwsCredentialProvider());
+        AmazonSimpleDB simpleDBClient = AmazonSimpleDBClient.builder().withCredentials(provider.getAwsCredentialProvider()).build();
 
         String nextToken = null;
         String appid = asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName;

priam/src/main/java/com/netflix/priam/SimpleDBConfigSource.java

@@ -26,7 +26,7 @@ public class IAMCredential implements ICredential
 
     public IAMCredential()
     {
-        this.iamCredProvider = new InstanceProfileCredentialsProvider();
+        this.iamCredProvider = InstanceProfileCredentialsProvider.getInstance();
     }
 
     public String getAccessKeyId()

priam/src/main/java/com/netflix/priam/aws/IAMCredential.java

@@ -15,6 +15,7 @@
  */
 package com.netflix.priam.aws;
 
+import com.amazonaws.services.s3.AmazonS3;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,7 +40,7 @@
 public class S3CrossAccountFileSystem  {
 	private static final Logger logger = LoggerFactory.getLogger(S3CrossAccountFileSystem.class);
 
-	private AmazonS3Client s3Client;
+	private AmazonS3 s3Client;
 	private S3FileSystem s3fs;
 	private IConfiguration config;
 	private IS3Credential s3Credential;
@@ -58,7 +59,7 @@ public IBackupFileSystem getBackupFileSystem() {
 		return this.s3fs;
 	}
 
-	public AmazonS3Client getCrossAcctS3Client() {
+	public AmazonS3 getCrossAcctS3Client() {
 		if (this.s3Client == null ) {
 
 			synchronized(this) {
@@ -67,13 +68,13 @@ public AmazonS3Client getCrossAcctS3Client() {
 
 					try {
 
-						this.s3Client = new AmazonS3Client(s3Credential.getAwsCredentialProvider());
+						this.s3Client = AmazonS3Client.builder().withCredentials(s3Credential.getAwsCredentialProvider()).withRegion(config.getDC()).build();//new AmazonS3Client(s3Credential.getAwsCredentialProvider());
 
 					} catch (Exception e) {
 						throw new IllegalStateException("Exception in getting handle to s3 client.  Msg: " + e.getLocalizedMessage(), e);
 
 					}
-					this.s3Client.setEndpoint(s3fs.getS3Endpoint(config));
+					//this.s3Client.setEndpoint(s3fs.getS3Endpoint(config));
 
 					//Lets leverage the IBackupFileSystem behaviors except we want it to use our amazon S3 client which has cross AWS account api capability.
 					this.s3fs.setS3Client(s3Client);

priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java

@@ -108,8 +108,8 @@ public S3EncryptedFileSystem(Provider<AbstractBackupPath> pathProvider, ICompres
         	throw new RuntimeException("Unable to regiser JMX bean: " + mbeanName + " to JMX server.  Msg: " + e.getLocalizedMessage(), e);
         }        
 
-        super.s3Client = new AmazonS3Client(cred.getAwsCredentialProvider());
-        super.s3Client.setEndpoint(super.getS3Endpoint(this.config));        
+        super.s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build();
+        //super.s3Client.setEndpoint(super.getS3Endpoint(this.config));
 	}
 
 	@Override

priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java

@@ -32,6 +32,7 @@
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
 
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.*;
 import com.netflix.priam.aws.auth.IS3Credential;
 import com.netflix.priam.backup.*;
@@ -105,8 +106,8 @@ public S3FileSystem(Provider<AbstractBackupPath> pathProvider, ICompression comp
             throw new RuntimeException(e);
         }        
 
-        super.s3Client = new AmazonS3Client(cred.getAwsCredentialProvider());
-        super.s3Client.setEndpoint(super.getS3Endpoint(this.config));
+        super.s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build();
+        //super.s3Client.setEndpoint(super.getS3Endpoint(this.config));
     }
 
     @Override
@@ -242,7 +243,7 @@ public void cleanup()
     /*
      * A means to change the default handle to the S3 client.
      */
-    public void setS3Client(AmazonS3Client client) {
+    public void setS3Client(AmazonS3 client) {
     	super.s3Client = client;
     }
 

priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java

@@ -60,23 +60,23 @@ public S3FileSystemBase (IMetricPublisher metricPublisher) {
         awsSlowDownMeasurement = new AWSSlowDownExceptionMeasurement(); //a counter of AWS warning for all uploads
     }
 
-    /*
-     * S3 End point information
-     * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
-     */
-    protected String getS3Endpoint(IConfiguration config)
-    {
-    	 final String curRegion = config.getDC();
-         if("us-east-1".equalsIgnoreCase(curRegion) ||
-            "us-west-1".equalsIgnoreCase(curRegion) ||
-            "us-west-2".equalsIgnoreCase(curRegion)	|| 
-            "eu-west-1".equalsIgnoreCase(curRegion) ||
-            "sa-east-1".equalsIgnoreCase(curRegion) ||
-            "eu-central-1".equalsIgnoreCase(curRegion))
-             return config.getS3EndPoint();
-         
-         throw new IllegalStateException("Unsupported region for this application: " + curRegion);
-    }
+//    /*
+//     * S3 End point information
+//     * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+//     */
+//    protected String getS3Endpoint(IConfiguration config)
+//    {
+//    	 final String curRegion = config.getDC();
+//         if("us-east-1".equalsIgnoreCase(curRegion) ||
+//            "us-west-1".equalsIgnoreCase(curRegion) ||
+//            "us-west-2".equalsIgnoreCase(curRegion)	||
+//            "eu-west-1".equalsIgnoreCase(curRegion) ||
+//            "sa-east-1".equalsIgnoreCase(curRegion) ||
+//            "eu-central-1".equalsIgnoreCase(curRegion))
+//             return config.getS3EndPoint();
+//
+//         throw new IllegalStateException("Unsupported region for this application: " + curRegion);
+//    }
 
     public AmazonS3 getS3Client()
     {

priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java

@@ -49,7 +49,7 @@ public class S3PrefixIterator implements Iterator<AbstractBackupPath>
     private String bucket = "";
     private String clusterPath = "";
     private SimpleDateFormat datefmt = new SimpleDateFormat("yyyyMMdd");
-    private ObjectListing objectListing;
+    private ObjectListing objectListing = null;
     Date date;
 
     @Inject
@@ -68,7 +68,6 @@ public S3PrefixIterator(IConfiguration config, Provider<AbstractBackupPath> path
         String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP));
         bucket = paths[0];
         this.clusterPath = remotePrefix(path);
-        objectListing = null;
         iterator = createIterator();
     }