Fixed upload of large files to AWS S3 (version 0.2.1)

Author: fdupoux

ChangeLog

@@ -2,7 +2,10 @@
 #                        ChangeLog for rubackup                              #
 ##############################################################################
 
-* 0.2.0
+* 0.2.1 (2015-12-01)
+  - Fixed upload of large files to AWS S3
+
+* 0.2.0 (2015-11-29)
   - Upgraded to AWS SDK version 2
 
 * 0.1.1 (2015-03-29):

rubackup.spec

@@ -33,6 +33,9 @@ rsync -a ${RPM_BUILD_DIR}/rubackup-%{version}/src/ ${RPM_BUILD_ROOT}%{deploydir}
 
 %changelog
 
+* Tue Dec 01 2015 Francois Dupoux - 0.2.1-1
+- Fixed upload of large files to AWS S3
+
 * Sun Nov 29 2015 Francois Dupoux - 0.2.0-1
 - Upgraded to AWS SDK version 2
 

src/lib/modules/remote-aws-s3.rb

@@ -46,9 +46,10 @@ def get_s3_access(remote_opts)
         keysecret = accesskey.fetch('secret')
         # get an instance of the S3 interface.
         creds = Aws::Credentials.new(keypublic, keysecret)
-        s3 = Aws::S3::Client.new(region: awsregion, credentials: creds)
-        # return s3 object
-        return s3
+        s3clt = Aws::S3::Client.new(region: awsregion, credentials: creds)
+        s3res = Aws::S3::Resource.new(:access_key_id => keypublic, :secret_access_key => keysecret, :region => awsregion)
+        # return s3 objects
+        return s3clt,s3res
     end
 
     # Return list of file names found in this bucket for this type of backup
@@ -58,8 +59,8 @@ def list(entrydata)
         basename = entrydata['bakfile_basename']
         bucketdata = remote_opts.fetch('s3_bucket')
         bucketname = bucketdata.fetch('bucket')
-        s3 = get_s3_access(remote_opts)
-        allobjs = s3.list_objects({bucket: bucketname}).contents # all objects including checksums files
+        s3clt,s3res = get_s3_access(remote_opts)
+        allobjs = s3clt.list_objects({bucket: bucketname}).contents # all objects including checksums files
         coreobj = allobjs.select { |obj| ((obj.key =~ /#{basename}-(\d{8})/) and (Checksum.checksum?(obj.key) == false)) }
         coreobj.each do |obj|
             bakfile = Bakfile.new
@@ -82,20 +83,18 @@ def send(entrydata, fullpath)
         remote_opts = entrydata.fetch('remote_opts')
         bucketdata = remote_opts.fetch('s3_bucket')
         bucketname = bucketdata.fetch('bucket')
-        s3 = get_s3_access(remote_opts)
+        s3clt,s3res = get_s3_access(remote_opts)
         filename = File.basename(fullpath)
-        File.open(fullpath, 'rb') do |file|
-          s3.put_object(bucket: bucketname, key: filename, body: file)
-        end
+        s3res.bucket(bucketname).object(filename).upload_file(fullpath)
     end
 
     # Delete a file in the bucket
     def delete(entrydata, filename)
         remote_opts = entrydata.fetch('remote_opts')
         bucketdata = remote_opts.fetch('s3_bucket')
         bucketname = bucketdata.fetch('bucket')
-        s3 = get_s3_access(remote_opts)
-        s3.delete_object( { bucket: bucketname, key: filename} )
+        s3clt,s3res = get_s3_access(remote_opts)
+        s3clt.delete_object( { bucket: bucketname, key: filename} )
     end
 
 end

src/rubackup.rb

@@ -26,7 +26,7 @@
 require 'yaml'
 
 # Define constants
-$RUBACKUP_VERSION = "0.2.0"
+$RUBACKUP_VERSION = "0.2.1"
 $RUBACKUP_MIN_RUBY = 1.9 # Required by Open3.pipeline()
 
 # Define global variables