1#!/usr/bin/env python 2import os 3from benchmarkBatch import generateBatchScript 4 5class PETSc(object): 6 def __init__(self): 7 return 8 9 def dir(self): 10 '''Return the root directory for the PETSc tree (usually $PETSC_DIR)''' 11 # This should search for a valid PETSc 12 return os.environ['PETSC_DIR'] 13 14 def arch(self): 15 '''Return the PETSc build label (usually $PETSC_ARCH)''' 16 # This should be configurable 17 return os.environ['PETSC_ARCH'] 18 19 def mpiexec(self): 20 '''Return the path for the mpi launch executable''' 21 mpiexec = os.path.join(self.dir(), self.arch(), 'bin', 'mpiexec') 22 if not os.path.isfile(mpiexec): 23 return None 24 return mpiexec 25 26 def example(self, num): 27 '''Return the path to the executable for a given example number''' 28 return os.path.join(self.dir(), self.arch(), 'lib', 'ex'+str(num)+'-obj', 'ex'+str(num)) 29 30class PETScExample(object): 31 def __init__(self, library, num, **defaultOptions): 32 self.petsc = PETSc() 33 self.library = library 34 self.num = num 35 self.opts = defaultOptions 36 return 37 38 @staticmethod 39 def runShellCommand(command, cwd = None): 40 import subprocess 41 42 Popen = subprocess.Popen 43 PIPE = subprocess.PIPE 44 print 'Executing: %s\n' % (command,) 45 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, bufsize=-1, shell=True, universal_newlines=True) 46 (out, err) = pipe.communicate() 47 ret = pipe.returncode 48 return (out, err, ret) 49 50 def optionsToString(self, **opts): 51 '''Convert a dictionary of options to a command line argument string''' 52 a = [] 53 for key,value in opts.iteritems(): 54 if value is None: 55 a.append('-'+key) 56 else: 57 a.append('-'+key+' '+str(value)) 58 return ' '.join(a) 59 60 def run(self, numProcs = 1, **opts): 61 if self.petsc.mpiexec() is None: 62 cmd = self.petsc.example(self.num) 63 else: 64 cmd = ' '.join([self.petsc.mpiexec(), '-n', str(numProcs), self.petsc.example(self.num)]) 65 cmd += ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts) 66 if 'batch' in opts and opts['batch']: 67 del opts['batch'] 68 filename = generateBatchScript(self.num, numProcs, 120, ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts)) 69 # Submit job 70 out, err, ret = self.runShellCommand('qsub -q gpu '+filename) 71 if ret: 72 print err 73 print out 74 else: 75 out, err, ret = self.runShellCommand(cmd) 76 if ret: 77 print err 78 print out 79 return 80 81def processSummary(moduleName, times, events): 82 '''Process the Python log summary into plot data''' 83 m = __import__(moduleName) 84 reload(m) 85 # Total Time 86 times.append(m.Time[0]) 87 # Common events 88 # VecMAXPY and VecMDot essentially give KSPGMRESOrthog 89 # Add the time and flop rate 90 for name in ['VecMDot', 'VecMAXPY', 'KSPGMRESOrthog', 'MatMult']: 91 if not name in events: 92 events[name] = [] 93 events[name].append((m.Solve.event[name].Time[0], m.Solve.event[name].Flops[0]/(m.Solve.event[name].Time[0] * 1e6))) 94 # Particular events 95 for name in ['VecCUSPCopyTo', 'VecCUSPCopyFrom', 'MatCUSPCopyTo']: 96 if name in m.Solve.event: 97 if not name in events: 98 events[name] = [] 99 events[name].append((m.Solve.event[name].Time[0], m.Solve.event[name].Flops[0]/(m.Solve.event[name].Time[0] * 1e6))) 100 return 101 102def plotSummaryLine(library, num, sizes, times, events): 103 from pylab import legend, plot, show, title, xlabel, ylabel 104 import numpy as np 105 showTime = False 106 showEventTime = True 107 showEventFlops = True 108 arches = sizes.keys() 109 # Time 110 if showTime: 111 data = [] 112 for arch in arches: 113 data.append(sizes[arch]) 114 data.append(times[arch]) 115 plot(*data) 116 title('Performance on '+library+' Example '+str(num)) 117 xlabel('Number of Dof') 118 ylabel('Time (s)') 119 legend(arches, 'upper left', shadow = True) 120 show() 121 # Common event time 122 # We could make a stacked plot like Rio uses here 123 if showEventTime: 124 data = [] 125 names = [] 126 for event, color in [('VecMDot', 'b'), ('VecMAXPY', 'g'), ('MatMult', 'r')]: 127 for arch, style in zip(arches, ['-', ':']): 128 names.append(arch+' '+event) 129 data.append(sizes[arch]) 130 data.append(np.array(events[arch][event])[:,0]) 131 data.append(color+style) 132 plot(*data) 133 title('Performance on '+library+' Example '+str(num)) 134 xlabel('Number of Dof') 135 ylabel('Time (s)') 136 legend(names, 'upper left', shadow = True) 137 show() 138 # Common event flops 139 # We could make a stacked plot like Rio uses here 140 if showEventFlops: 141 data = [] 142 names = [] 143 for event, color in [('VecMDot', 'b'), ('VecMAXPY', 'g'), ('MatMult', 'r')]: 144 for arch, style in zip(arches, ['-', ':']): 145 names.append(arch+' '+event) 146 data.append(sizes[arch]) 147 data.append(np.array(events[arch][event])[:,1]) 148 data.append(color+style) 149 plot(*data) 150 title('Performance on '+library+' Example '+str(num)) 151 xlabel('Number of Dof') 152 ylabel('Computation Rate (MF/s)') 153 legend(names, 'upper left', shadow = True) 154 show() 155 return 156 157def plotSummaryBar(library, num, sizes, times, events): 158 import numpy as np 159 import matplotlib.pyplot as plt 160 161 eventNames = ['VecMDot', 'VecMAXPY', 'MatMult'] 162 eventColors = ['b', 'g', 'r'] 163 arches = sizes.keys() 164 names = [] 165 N = len(sizes[arches[0]]) 166 width = 0.2 167 ind = np.arange(N) - 0.25 168 bars = {} 169 for arch in arches: 170 bars[arch] = [] 171 bottom = np.zeros(N) 172 for event, color in zip(eventNames, eventColors): 173 names.append(arch+' '+event) 174 times = np.array(events[arch][event])[:,0] 175 bars[arch].append(plt.bar(ind, times, width, color=color, bottom=bottom)) 176 bottom += times 177 ind += 0.3 178 179 plt.xlabel('Number of Dof') 180 plt.ylabel('Time (s)') 181 plt.title('GPU vs. CPU Performance on '+library+' Example '+str(num)) 182 plt.xticks(np.arange(N), map(str, sizes[arches[0]])) 183 #plt.yticks(np.arange(0,81,10)) 184 #plt.legend( (p1[0], p2[0]), ('Men', 'Women') ) 185 plt.legend([bar[0] for bar in bars[arches[0]]], eventNames, 'upper right', shadow = True) 186 187 plt.show() 188 return 189 190if __name__ == '__main__': 191 library = 'SNES' 192 num = 19 193 ex = PETScExample(library, num, pc_type='none', dmmg_nlevels=1, log_summary='summary.dat', log_summary_python='summary.py', mat_no_inode=None, preload='off') 194 sizes = {} 195 times = {} 196 events = {} 197 for name, vecType, matType, opts in [('CPU', 'seq', 'seqaij', {}), ('GPU', 'seqcusp', 'seqaijcusp', {'cusp_synchronize': None})]: 198 sizes[name] = [] 199 times[name] = [] 200 events[name] = {} 201 #for n in [10, 20, 50, 100, 150, 200]: 202 for n in [10, 20]: 203 ex.run(da_grid_x=n, da_grid_y=n, da_vec_type=vecType, da_mat_type=matType, **opts) 204 sizes[name].append(n*n * 4) 205 processSummary('summary', times[name], events[name]) 206 plotSummaryLine(library, num, sizes, times, events) 207