aging release

benchmarks/aging/mailserver/config.ini

@@ -0,0 +1,11 @@
+[Parameters]
+workingpartition = /dev/sdb1
+workingdir = /mnt/benchmark
+fstype = ext4
+checkoutspergrep = 1000
+greprandom = 8TCg8BVMrUz4xoaU
+totalcheckouts = 60001
+pretestremount = True
+writeamplength = 100
+cleantest = True
+

benchmarks/aging/mailserver/mailserver-aging.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python
+import imaplib
+import sys
+import random
+import os
+import threading
+import time
+import types
+import subprocess
+
+SERVER = "localhost"
+USER = ["ftfstest1", "ftfstest2", "ftfstest3", "ftfstest4", "ftfstest5", "ftfstest6", "ftfstest7", "ftfstest8", "ftfstest9", "ftfstest10", "ftfstest11", "ftfstest12", "ftfstest13", "ftfstest14", "ftfstest15", "ftfstest16"]
+PASS = ["oscarlab", "oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab"]
+
+n_user = 2
+n_box = 80
+boxsize = 1000
+max_msg_len = 32768
+run_time = 1800
+n_top = 8000
+
+def worker_thread(i, n_op_thread, running) :
+	m = imaplib.IMAP4_SSL(SERVER)
+	m.login(USER[i], PASS[i])
+	while not running.isSet() :
+		pass
+	n_ops = [0] * 3
+#	while running.isSet() :
+	for i in range(n_top) :
+		boxnum = random.randint(1, n_box) - 1
+		box = "box%d" % boxnum
+		x = m.select(box)
+		rand_op = random.randint(1, 2) - 1
+		if rand_op == 0 :
+			msg_len = random.randint(1, max_msg_len)
+			msg = os.urandom(msg_len)
+			m.APPEND(box, None, None, msg)
+		else : 
+			typ, msg_ids = m.search(None, 'ALL')
+			msgs = msg_ids[0].split()
+			msg_num = random.randint(1, len(msgs)) - 1
+			msg = msgs[msg_num]
+#			if rand_op == 1 :
+			m.store(msg, "+FLAGS", "(\\Deleted)")
+			m.expunge()
+#			else :
+#				typ, data = m.fetch(msg, "(RFC822 FLAGS)")
+#				flagged = 0
+#				if type(data[0]) is types.NoneType :
+#					continue
+#				flagged = 0
+#				for flag in imaplib.ParseFlags(data[0][0]) :
+#					if (flag == "\Flagged") :
+#						flagged = 1
+#				if flagged :
+#					m.store(msg, "-FLAGS", "(\\FLAGGED)")
+#				else :
+#					m.store(msg, "+FLAGS", "(\\FLAGGED)")
+		n_ops[rand_op] = n_ops[rand_op] + 1
+	subprocess.call('echo "flush" > /proc/toku_flusher', shell=True)
+	m.logout()
+	print "Thread %d: append %d delete %d flag change %d" % (i, n_ops[0], n_ops[1], n_ops[2])
+	n_op_thread.append(n_ops[0] + n_ops[1] + n_ops[2])
+
+print "MAILSERVER AGEING"
+f=open('mailservertime.out','a')
+
+t = []
+running = threading.Event()
+n_op_thread = []
+for i in range(n_user) :
+	tmp_t = threading.Thread(target = worker_thread, args = (i, n_op_thread, running,))
+	tmp_t.start()
+	t.append(tmp_t)
+
+time.sleep(2)
+running.set()
+t1 = time.time()
+#time.sleep(run_time)
+#running.clear()
+for i in range(n_user):
+	t[i].join()
+t2 = time.time()
+n_op_total = 0
+for i in range(n_user) :
+	n_op_total = n_op_total + n_op_thread[i]
+print "This experiment took %f seconds" % (t2 - t1)
+print "%d ops are executed (%f op/s)" % (n_op_total, n_op_total / (t2 - t1))
+f.write("Time\t")
+f.write(str(t2 - t1) + '\t')
+f.write("Nops\t")
+f.write(str(n_op_total) + '\n')
+
+sys.exit(0)

benchmarks/aging/mailserver/mailserver-setup.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+import imaplib
+import sys
+import random
+import os
+
+SERVER = "localhost"
+USER = ["ftfstest1", "ftfstest2", "ftfstest3", "ftfstest4", "ftfstest5", "ftfstest6", "ftfstest7", "ftfstest8", "ftfstest9", "ftfstest10", "ftfstest11", "ftfstest12", "ftfstest13", "ftfstest14", "ftfstest15", "ftfstest16"]
+PASS = ["oscarlab", "oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab","oscarlab"]
+
+n_user = 2
+n_box = 80
+boxsize = 1000
+max_msg_len = 32768
+
+print "MAILSERVER setup"
+
+for i in range(n_user) :
+	if 0 == os.fork() :
+		# Child i
+		random.seed()
+		M = imaplib.IMAP4_SSL(SERVER)
+		M.login(USER[i], PASS[i])
+		for j in range(n_box) :
+			box = "box%d" % j
+			M.delete(box)
+			M.create(box)
+			print "MAILSERVER setup: child %d box %d" % (i, j)
+			for k in range(boxsize) :
+				msg_len = random.randint(1, max_msg_len)
+				msg = os.urandom(msg_len)
+				M.append(box, None, None, msg)
+		M.logout()
+		sys.exit(0)
+for i in range(n_user) :
+	os.wait()
+sys.exit(0)

benchmarks/aging/mailserver/mailserver-setupdir.sh

@@ -0,0 +1,12 @@
+#/bin/bash
+#run as root
+
+for i in $(seq 2)
+do
+	dir=/mnt/benchmark/ftfstest${i}/Maildir
+	sudo mkdir -p ${dir}
+	sudo chown -R ftfstest${i}:ftfstest${i} /mnt/benchmark/ftfstest${i}
+done
+
+sudo mkdir -p traces
+sudo mkdir -p clean_temp

benchmarks/aging/mailserver/resultify.py

@@ -0,0 +1,53 @@
+#resultify.py outputs results to csv
+
+import csv
+import json
+import subprocess
+
+with open('results.json') as datafile:
+ data = json.load(datafile)
+
+resultfile = open('results.csv','wb')
+resultwriter = csv.writer(resultfile,delimiter=',')
+resultwriter.writerow( ('FS Size', 'Aged Time', 'Clean Time', 'Aged Total Seek Distance', 'Clean Total Seek Distance') )
+
+for i in range(0,len(data)):
+ print('Processing test ' + str(i))
+ #Compute seeks and sectors read form blktrace
+ subprocess.call('blkparse -q -a issue -f "%S %n \n" -i trace' + str(i).zfill(6) + '.blktrace.0 > agedgrep.txt', cwd='traces', shell=True)
+ totalseekdistance = 0
+ #totalsectors = 0
+ with open('traces/agedgrep.txt') as tracefile:
+  lastsector = -1
+  for line in tracefile:
+    splitline = line.split()
+    if splitline[0].isdigit():
+     sector = int(splitline[0])
+     numblocks = int(splitline[1])
+     totalseekdistance = totalseekdistance + abs(sector - lastsector)
+     #if lastsector != sector:
+     # seeks = seeks + 1
+     #totalsectors = totalsectors + numblocks
+     lastsector = sector + numblocks
+
+ #compute clean reads and sectors read from blktrace
+ subprocess.call('blkparse -q -a issue -f "%S %n \n" -i cleantrace' + str(i).zfill(6) + '.blktrace.0 > cleangrep.txt', cwd='traces', shell=True)
+ cleanseekdistance = 0
+ #cleantotalsectors = 0
+ with open('traces/cleangrep.txt') as tracefile:
+  lastsector = -1
+  for line in tracefile:
+   splitline = line.split()
+   if splitline[0].isdigit():
+    sector = int(splitline[0])
+    numblocks = int(splitline[1])
+    cleanseekdistance = cleanseekdistance + abs(sector - lastsector)
+    #if lastsector != sector:
+    # cleanseeks = cleanseeks + 1
+    #cleantotalsectors = cleantotalsectors + numblocks
+    lastsector = sector + numblocks
+
+ resultrow = (data[i] + [totalseekdistance, cleanseekdistance])
+ resultwriter.writerow(resultrow)
+
+ subprocess.call('sudo rm -f *.txt', cwd='traces', shell=True) 

benchmarks/aging/mailserver/run-all.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+#run-all.py
+#Improved version of the aging grep test with blktrace, backups, and config files. Running without a config file will generate one with default values.
+
+import time
+import subprocess
+from subprocess import CalledProcessError
+import ConfigParser
+import os.path
+import sys
+
+#Global variables: parameters for the test. Assigned by initialize().
+WorkingDir = ''
+FSType = ''
+CleanTest = True
+
+#initialize checks if there is a config file, and reads it if it exists. If no config file exists, it creates one with default values.
+def initialize():
+ Config = ConfigParser.ConfigParser()
+ if os.path.isfile('config.ini'):
+  print('Config file found; reading values.')
+  Config.read('config.ini')
+  try:
+   global WorkingDir
+   WorkingDir = Config.get('Parameters','WorkingDir')
+   global FSType
+   FSType = Config.get('Parameters','FSType')
+   global CleanTest
+   CleanTest = Config.getboolean('Parameters','CleanTest')
+  except:
+   print('An exception occured while reading the config file.')
+   sys.exit()
+  print('Configuration loaded.')
+ return()
+
+#subprocess.check_call('python populate-conf.py', shell=True)
+
+initialize()
+
+#make sure blktrace isn't running
+try:
+ subprocess.call('kill -15 ' + subprocess.check_output(["pidof","-s","blktrace"]), shell=True)
+except Exception, e:
+ pass
+
+#mount fs
+if FSType == 'betrfs':
+ subprocess.check_call( '../../setup-ftfs.sh', shell=True)
+else:
+ subprocess.check_call( '../../setup-' + FSType +'.sh', shell=True)
+
+subprocess.check_call('sudo dovecot', shell=True)
+#check if benchmark exists
+#setup repos
+if os.path.isdir('benchmark') == False:
+ subprocess.call('sudo bash mailserver-setupdir.sh', shell=True)
+ subprocess.call('sudo python mailserver-setup.py', shell=True)
+else:
+ print 'copying benchmark dir over'
+ subprocess.call('cp -a benchmark/* /mnt/benchmark', shell=True) 
+ subprocess.call('chown -R ftfstest1:ftfstest1 /mnt/benchmark/ftfstest1', shell=True) 
+ subprocess.call('chown -R ftfstest2:ftfstest2 /mnt/benchmark/ftfstest2', shell=True) 
+#setup test always
+subprocess.check_call('sudo python runtest.py ', shell=True)
+#run cleantest.py
+if CleanTest == True:
+ #mount clean fs
+ subprocess.check_call('python runcleantest.py', shell=True)
+subprocess.check_call('sudo dovecot stop', shell=True)

benchmarks/aging/mailserver/runcleantest.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+#test.py
+#Improved version of the aging grep test with blktrace, backups, and config files. Running without a config file will generate one with default values.
+
+import time
+import subprocess
+import ConfigParser
+import os.path
+import sys
+import json
+import argparse
+
+#Global variables: parameters for the test. Assigned by initialize().
+WorkingDir = ''
+WorkingPartition = ''
+GREPRandom = ''
+CheckoutsPerGREP = 0
+GREPsPerBackup = 0
+TotalCheckouts = 0
+FSType = ''
+PreTestRemount = True
+WriteAmpLength = 0
+
+#initialize checks if there is a config file, and reads it if it exists. If no config file exists, it creates one with default values.
+def initialize():
+ Config = ConfigParser.ConfigParser()
+ if os.path.isfile('config.ini'):
+  print('Config file found; reading values.')
+  Config.read('config.ini')
+  try:
+   global WorkingDir
+   WorkingDir = Config.get('Parameters','WorkingDir')
+   global WorkingPartition
+   WorkingPartition = Config.get('Parameters','WorkingPartition')
+   global GREPRandom
+   GREPRandom = Config.get('Parameters','GREPRandom')
+   global CheckoutsPerGREP
+   CheckoutsPerGREP = Config.getint('Parameters','CheckoutsPerGREP')
+   global TotalCheckouts
+   TotalCheckouts = Config.getint('Parameters','TotalCheckouts')  
+   global FSType
+   FSType = Config.get('Parameters','FSType')
+   global PreTestRemount
+   PreTestRemount = Config.getboolean('Parameters','PreTestRemount')
+   global WriteAmpLength
+   WriteAmpLength = Config.getint('Parameters','WriteAmpLength')
+  except:
+   print('An exception occured while reading the config file.')
+   sys.exit()
+  print('Configuration loaded.')
+
+def cleangreptest(testno):
+ subprocess.check_call('rm -rf *', cwd='clean_temp', shell=True)
+ subprocess.check_call('cp -a ' + WorkingDir + '/* clean_temp', shell=True)
+ subprocess.check_call( '../../cleanup-fs.sh' , shell=True)
+ if FSType == 'betrfs':
+  subprocess.check_call( '../../setup-ftfs.sh', shell=True)
+ else:
+  subprocess.check_call( '../../setup-' + FSType + '.sh', shell=True)
+ subprocess.check_call('cp -a clean_temp/* ' + WorkingDir, shell=True)
+ if(PreTestRemount):
+  if(FSType=='betrfs'):
+   subprocess.check_call( '../../cleanup-fs.sh' , shell=True)
+   subprocess.check_call( '../../mount-ftfs.sh' , shell=True)
+  elif(FSType=='zfs'):
+   subprocess.call('free && sync && echo 3 > /proc/sys/vm/drop_caches && free', shell=True)
+   subprocess.check_call( 'umount ' + WorkingDir, shell=True)
+   subprocess.check_call('zfs mount -a', shell=True)
+  else:
+   subprocess.check_call( '../../cleanup-fs.sh' , shell=True)
+   subprocess.check_call('mount ' + WorkingPartition + ' ' + WorkingDir, shell=True)
+ else:
+  subprocess.call('free && sync && echo 3 > /proc/sys/vm/drop_caches && free', shell=True)
+ subprocess.call('blktrace -a read -d ' + WorkingPartition.rstrip('0123456789') + ' -o traces/cleantrace' + str(testno).zfill(6) + ' &', shell=True)
+ start = time.time()
+ subprocess.call('(grep -r --binary-files=text "' + GREPRandom  + '" ' + WorkingDir +') > /dev/null', stderr=subprocess.STDOUT, shell=True)
+ stop = time.time() 
+ subprocess.call('kill -15 ' + subprocess.check_output(["pidof","-s","blktrace"]), shell=True)
+ return (stop - start)
+
+##################################################################################
+# Main Script Begins Here
+
+initialize()
+results= []
+
+for i in range(100): 
+ subprocess.check_call('sudo python mailserver-aging.py', shell=True)
+ print('Running clean grep test: ' + str(i))
+ thisgrep = cleangreptest(i)
+ print('clean grep test completed in ' + str(thisgrep) + ' seconds')
+ results.append(thisgrep)
+ with open('cleanresults.json','w') as outfile:
+  json.dump(results, outfile)
+ if(PreTestRemount):
+  if(FSType=='betrfs'):
+   subprocess.check_call( '../../cleanup-fs.sh' , shell=True)
+   subprocess.check_call( '../../mount-ftfs.sh' , shell=True)
+  elif(FSType=='zfs'):
+   subprocess.call('free && sync && echo 3 > /proc/sys/vm/drop_caches && free', shell=True)
+   subprocess.check_call( 'umount ' + WorkingDir, shell=True)
+   subprocess.check_call('zfs mount -a', shell=True)
+  else:
+   subprocess.check_call( '../../cleanup-fs.sh' , shell=True)
+   subprocess.check_call('mount ' + WorkingPartition + ' ' + WorkingDir, shell=True)
+ else:
+  subprocess.call('free && sync && echo 3 > /proc/sys/vm/drop_caches && free', shell=True)
+ time.sleep(1)
+ subprocess.call('blktrace -a write -d ' + WorkingPartition.rstrip('0123456789') + ' -o traces/cleanwritetrace' + str(i).zfill(6) + ' &', shell=True)
+ subprocess.call('kill -15 ' + subprocess.check_output(["pidof","-s","blktrace"]), shell=True)
+ time.sleep(1)
+
+try:
+ subprocess.call('kill -15 ' + subprocess.check_output(["pidof","-s","blktrace"]), shell=True)
+except Exception, e:
+ pass