SYS-471 bug-fixes in inject / calc_sums / verify

Dockerfile

@@ -19,7 +19,7 @@ ENV CRON_HOUR=0,8,16 \
     TZ=UTC
 
 ARG RSNAPSHOT_VERSION=1.4.3-r0
-ARG SECONDSHOT_VERSION=0.10.3
+ARG SECONDSHOT_VERSION=0.10.4
 ARG RRSYNC_SHA=f7b931e73e811f76e0ad8466e654e374ee18025b837ec69abed805ff34e0f1ef
 
 VOLUME /backups /metadata /etc/secondshot/conf.d

bin/check_rsnap.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # $Id: check_rsnap.py,v 1.3 2013/03/31 16:46:46 root Exp $
 #  check_rsnap.py
 #
@@ -20,110 +20,97 @@
 #  MA 02110-1301, USA.
 
 import sys
-import time
 import argparse
-import MySQLdb
+import pymysql
+
 
 def argument_parser():
     parser = argparse.ArgumentParser(description='Check db for rsnap backups')
-    parser.add_argument('-H',
-        help = 'host to query (Default: all)',
-        dest = 'host',
-        type = str,
-        default = 'all'
-        )
-    parser.add_argument('-w',
-        help = 'WARNING if number of backups too low (Default: 1)',
-        dest = 'warning',
-        type = int,
-        default = 1
-        )
-    parser.add_argument('-c',
-        help = 'CRITICAL if number of backups too low (Default: 1)',
-        dest = 'critical',
-        type = int,
-        default = 1
-        )
-    parser.add_argument('-f',
-        help = 'WARNING if number of files too low (Default: 1000)',
-        dest = 'file_warning',
-        type = int,
-        default = 1000
-        )
-    parser.add_argument('-i',
-        help = 'backup interval, in hours (Default: 24)',
-        dest = 'interval',
-        type = int,
-        default = '24'
-        )
-    parser.add_argument('-d',
-        help = 'database host (Default: localhost)',
-        dest = 'dbhost',
-        type = str,
-        default = 'localhost'
-        )
-    parser.add_argument('-P',
-        help = 'database port (Default: 3306)',
-        dest = 'dbport',
-        type = int,
-        default = '3306'
-        )
-    parser.add_argument('-n',
-        help = 'name of database (Default: rsnap)',
-        dest = 'dbname',
-        type = str,
-        default = 'rsnap'
-        )
-    parser.add_argument('-u',
-        help = 'database user (Default: nagmon)',
-        dest = 'dbuser',
-        type = str,
-        default = 'nagmon'
-        )
-    parser.add_argument('-p',
-		    help = 'database password',
-        dest = 'dbpass',
-        type = str
-        )
+    parser.add_argument(
+        '-H', help='host to query (Default: all)',
+        dest='host',
+        type=str, default='all')
+    parser.add_argument(
+        '-w', help='WARNING if number of backups too low (Default: 1)',
+        dest='warning',
+        type=int, default=1)
+    parser.add_argument(
+        '-c', help='CRITICAL if number of backups too low (Default: 1)',
+        dest='critical',
+        type=int, default=1)
+    parser.add_argument(
+        '-f', help='WARNING if number of files too low (Default: 1000)',
+        dest='file_warning',
+        type=int, default=1000)
+    parser.add_argument(
+        '-i', help='backup interval, in hours (Default: 24)',
+        dest='interval',
+        type=int, default='24')
+    parser.add_argument(
+        '-d', help='database host (Default: localhost)',
+        dest='dbhost',
+        type=str, default='localhost')
+    parser.add_argument(
+        '-P', help='database port (Default: 3306)',
+        dest='dbport',
+        type=int, default='3306')
+    parser.add_argument(
+        '-n', help='name of database (Default: rsnap)',
+        dest='dbname',
+        type=str, default='rsnap')
+    parser.add_argument(
+        '-u', help='database user (Default: nagmon)',
+        dest='dbuser',
+        type=str, default='nagmon')
+    parser.add_argument(
+        '-p', help='database password',
+        dest='dbpass',
+        type=str)
     return parser.parse_args()
 
+
 def main():
     args = argument_parser()
-    start_time = time.time()
-    db = MySQLdb.connect(host=args.dbhost, port=args.dbport, user=args.dbuser, passwd=args.dbpass, db=args.dbname)
+    db = pymysql.connect(host=args.dbhost, port=args.dbport, user=args.dbuser,
+                         password=args.dbpass, db=args.dbname)
     cur = db.cursor()
 
     # Look for recent backups
-    q = 'SELECT COUNT(*) FROM savesets WHERE finished > (NOW() - INTERVAL ' + str(args.interval) + ' HOUR)'
+    q = ('SELECT COUNT(*) FROM savesets WHERE finished > (NOW() - INTERVAL ' +
+         str(args.interval) + ' HOUR)')
     if args.host != 'all':
-        q = q + ' AND host_id=(SELECT id FROM hosts WHERE hostname="' + args.host + '" LIMIT 1)'
+        q += (' AND host_id=(SELECT id FROM hosts WHERE hostname="%s" LIMIT 1)'
+              % args.host)
     cur.execute(q)
     row = cur.fetchone()
     num_backups = row[0]
 
     # Count files in backups
-    q = 'SELECT files FROM savesets JOIN hosts on hosts.id=savesets.host_id \
-         WHERE finished > (now() - INTERVAL ' + str(args.interval) + ' HOUR)'
+    q = ('SELECT SUM(files) FROM savesets '
+         'JOIN hosts ON hosts.id=savesets.host_id '
+         'WHERE finished > (now() - INTERVAL ' + str(args.interval) + ' HOUR)')
     if args.host != 'all':
-        q = q + ' AND hostname="' + args.host + '"'
+        q += ' AND hostname="' + args.host + '"'
     cur.execute(q)
     row = cur.fetchone()
-    num_files = row[0]
+    num_files = row[0] if row else 0
 
     rsnap_stat = 'OK: rsnap'
     exit_value = 0
     if num_backups < args.critical:
         rsnap_stat = 'CRITICAL: rsnap'
-	exit_value = 2
+        exit_value = 2
     elif num_backups < args.warning:
         rsnap_stat = 'WARNING: rsnap'
-	exit_value = 1
-    elif num_files < args.file_warning:
+        exit_value = 1
+    elif num_files is None or num_files < args.file_warning:
         rsnap_stat = 'WARNING: rsnap'
         exit_value = 1
-    sys.stdout.write(rsnap_stat + " backups:" + str(num_backups) + " files:" + str(num_files) + "\n")
+    sys.stdout.write(rsnap_stat + " backups:" + str(num_backups) + " files:" +
+                     str(num_files) + "\n")
     sys.exit((exit_value))
 
+
 if __name__ == '__main__':
     main()
     sys.stdout.write("Something really bad happened!")

secondshot/_version.py

@@ -1 +1 @@
-__version__ = '0.10.3'
+__version__ = '0.10.4'

secondshot/actions.py

@@ -104,39 +104,42 @@ def calc_sums(self, saveset_id):
         except Exception as ex:
             Syslog.logger.warn('action=calc_sums msg=%s' % str(ex))
             Syslog.logger.traceback(ex)
-        if (not host or not location):
+        if not host or not location:
             sys.exit('action=calc_sums msg=missing host/location')
         Syslog.logger.info('START action=calc_sums saveset=%s from host=%s '
                            'for location=%s' % (saveset, host, location))
         manifest_file = os.path.join(
-            Config.snapshot_root, location, Config.manifest)
+            Config.snapshot_root, location, host, Config.manifest)
         with open(manifest_file, 'r+') as mfile:
             mfile.readline()
             position = mfile.tell()
             (numbytes, count, total) = (0, 0, 0)
-            for line in mfile:
+            for line in iter(mfile.readline, ''):
                 file_id, file_type, size, has_sum = line.strip().split(',')
                 total += int(size)
-                if has_sum == 'Y' or file_type != 'f' or int(size) == 0:
-                    continue
-                count += 1
-                try:
-                    file = self.session.query(File).filter_by(id=file_id).one()
-                    filename = os.path.join(
-                        Config.snapshot_root, location, file.path,
-                        file.filename)
-                    file.shasum = self._filehash(filename, Config.hashtype)
-                    self.session.add(file)
-                    mfile.seek(position)
-                    mfile.write('%s,%s,%s,Y\n' % (file_id, file_type, size))
-                    numbytes += file.size
-                except Exception as ex:
-                    Syslog.logger.warn('action=calc_sums id=%d msg=skipped '
-                                       'error=%s' % (file.id, str(ex)))
-                if (count % 1000 == 0):
-                    Syslog.logger.debug('action=calc_sums count=%d bytes=%d' %
-                                        (count, numbytes))
-                    self.session.commit()
+                if has_sum != 'Y' and file_type == 'f' and int(size) > 0:
+                    count += 1
+                    try:
+                        file = self.session.query(File).filter_by(
+                            id=file_id).one()
+                        filename = os.path.join(
+                            Config.snapshot_root, location, file.path,
+                            file.filename)
+                        file.shasum = self._filehash(filename, Config.hashtype)
+                        self.session.add(file)
+                        mfile.seek(position)
+                        mfile.write(
+                            '%s,%s,%s,Y\n' % (file_id, file_type, size))
+                        numbytes += file.size
+                    except Exception as ex:
+                        Syslog.logger.warn(
+                            'action=calc_sums id=%d msg=skipped error=%s'
+                            % (file.id, str(ex)))
+                    if (count % 1000 == 0):
+                        Syslog.logger.debug(
+                            'action=calc_sums count=%d bytes=%d'
+                            % (count, numbytes))
+                        self.session.commit()
                 position = mfile.tell()
 
         self.session.commit()
@@ -165,20 +168,22 @@ def inject(self, host, volume, pathname, saveset_id):
         except Exception as ex:
             sys.exit('action=inject Invalid host or volume: %s' % str(ex))
 
-        mfile = open(os.path.join(pathname, Config.manifest), 'w')
+        mfile = open(os.path.join(pathname, host,
+                                  Config.manifest), 'w')
         mfile.write('file_id,type,file_size,has_checksum\n')
 
         (count, numbytes, skipped) = (0, 0, 0)
-        for dirpath, _, filenames in os.walk(pathname):
+        for dirpath, _, filenames in os.walk(os.path.join(
+                pathname, host)):
             for filename in filenames:
                 if filename == Config.manifest:
                     continue
                 try:
                     stat = os.lstat(os.path.join(dirpath, filename))
                     _path = pymysql.escape_string(os.path.relpath(
-                        dirpath, Config.snapshot_root + '/' +
-                        Constants.SYNC_PATH).encode(
-                        'utf8', 'surrogateescape').decode('utf8'))
+                        dirpath, os.path.join(
+                            Config.snapshot_root, Constants.SYNC_PATH)).encode(
+                                'utf8', 'surrogateescape').decode('utf8'))
                     _filename = pymysql.escape_string(filename.encode(
                         'utf8', 'surrogateescape').decode('utf8'))
                 except OSError as ex:
@@ -507,7 +512,8 @@ def verify(self, savesets):
                 raise RuntimeError('VERIFY saveset=%s not found' % saveset)
 
             manifest_file = os.path.join(
-                Config.snapshot_root, record.location, Config.manifest)
+                Config.snapshot_root, record.location, record.host.hostname,
+                Config.manifest)
             mfile = open(manifest_file, 'r')
             mfile.readline()
             count, errors, missing, skipped = (0, 0, 0, 0)

tests/test_actions.py

@@ -73,7 +73,9 @@ def tearDown(self):
         super(TestActions, self).tearDown()
 
         os.remove(self.rsnapshot_conf)
+        """
         shutil.rmtree(self.snapshot_root)
+        """
         shutil.rmtree(self.testdata_path)
 
     @mock.patch('secondshot.syslogger.Syslog._now')
@@ -101,7 +103,7 @@ def test_inject(self):
 
         count = 0
         with open(os.path.join(
-                self.volume_path,
+                self.volume_path, self.testhost,
                 Constants.OPTS_DEFAULTS['manifest']), 'r') as mfile:
             headers = mfile.readline()
             self.assertEqual(headers, 'file_id,type,file_size,has_checksum\n')
@@ -133,7 +135,7 @@ def test_calc_sums(self):
 
         count = 0
         with open(os.path.join(
-                self.volume_path,
+                self.volume_path, self.testhost,
                 Constants.OPTS_DEFAULTS['manifest']), 'r') as mfile:
             headers = mfile.readline()
             self.assertEqual(headers, 'file_id,type,file_size,has_checksum\n')
@@ -172,7 +174,7 @@ def test_rotate(self, mock_subprocess):
             os.path.join(self.snapshot_root, 'short.0', self.testhost))
         obj = Actions(self.cli, db_engine=self.engine, db_session=self.session)
         obj.inject(self.testhost, self.volume,
-                   os.path.join(self.snapshot_root, 'short.0', self.testhost),
+                   os.path.join(self.snapshot_root, 'short.0'),
                    saveset.id)
         ret = obj.rotate('short')
         self.assertEqual(ret, expected)