1#!/usr/bin/env python 2import os,sys 3from benchmarkBatch import generateBatchScript 4 5class PETSc(object): 6 def __init__(self): 7 return 8 9 def dir(self): 10 '''Return the root directory for the PETSc tree (usually $PETSC_DIR)''' 11 # This should search for a valid PETSc 12 return os.environ['PETSC_DIR'] 13 14 def arch(self): 15 '''Return the PETSc build label (usually $PETSC_ARCH)''' 16 # This should be configurable 17 return os.environ['PETSC_ARCH'] 18 19 def mpiexec(self): 20 '''Return the path for the mpi launch executable''' 21 mpiexec = os.path.join(self.dir(), self.arch(), 'bin', 'mpiexec') 22 if not os.path.isfile(mpiexec): 23 return None 24 return mpiexec 25 26 def example(self, num): 27 '''Return the path to the executable for a given example number''' 28 return os.path.join(self.dir(), self.arch(), 'lib', 'ex'+str(num)+'-obj', 'ex'+str(num)) 29 30 def source(self, library, num): 31 '''Return the path to the sources for a given example number''' 32 d = os.path.join(self.dir(), 'src', library.lower(), 'examples', 'tutorials') 33 name = 'ex'+str(num) 34 sources = [] 35 for f in os.listdir(d): 36 if f == name+'.c': 37 sources.append(f) 38 elif f.startswith(name) and f.endswith('.cu'): 39 sources.append(f) 40 return map(lambda f: os.path.join(d, f), sources) 41 42class PETScExample(object): 43 def __init__(self, library, num, **defaultOptions): 44 self.petsc = PETSc() 45 self.library = library 46 self.num = num 47 self.opts = defaultOptions 48 return 49 50 @staticmethod 51 def runShellCommand(command, cwd = None): 52 import subprocess 53 54 Popen = subprocess.Popen 55 PIPE = subprocess.PIPE 56 print 'Executing: %s\n' % (command,) 57 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, bufsize=-1, shell=True, universal_newlines=True) 58 (out, err) = pipe.communicate() 59 ret = pipe.returncode 60 return (out, err, ret) 61 62 def optionsToString(self, **opts): 63 '''Convert a dictionary of options to a command line argument string''' 64 a = [] 65 for key,value in opts.iteritems(): 66 if value is None: 67 a.append('-'+key) 68 else: 69 a.append('-'+key+' '+str(value)) 70 return ' '.join(a) 71 72 def run(self, numProcs = 1, **opts): 73 if self.petsc.mpiexec() is None: 74 cmd = self.petsc.example(self.num) 75 else: 76 cmd = ' '.join([self.petsc.mpiexec(), '-n', str(numProcs), self.petsc.example(self.num)]) 77 cmd += ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts) 78 if 'batch' in opts and opts['batch']: 79 del opts['batch'] 80 filename = generateBatchScript(self.num, numProcs, 120, ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts)) 81 # Submit job 82 out, err, ret = self.runShellCommand('qsub -q gpu '+filename) 83 if ret: 84 print err 85 print out 86 else: 87 out, err, ret = self.runShellCommand(cmd) 88 if ret: 89 print err 90 print out 91 return out 92 93def processSummary(moduleName, defaultStage, eventNames, times, events): 94 '''Process the Python log summary into plot data''' 95 m = __import__(moduleName) 96 reload(m) 97 # Total Time 98 times.append(m.Time[0]) 99 # Particular events 100 for name in eventNames: 101 if name.find(':') >= 0: 102 stageName, name = name.split(':', 1) 103 stage = getattr(m, stageName) 104 else: 105 stage = getattr(m, defaultStage) 106 if name in stage.event: 107 if not name in events: 108 events[name] = [] 109 events[name].append((stage.event[name].Time[0], stage.event[name].Flops[0]/(stage.event[name].Time[0] * 1e6))) 110 return 111 112def plotSummaryLine(library, num, sizes, times, events): 113 from pylab import legend, plot, show, title, xlabel, ylabel 114 import numpy as np 115 showTime = False 116 showEventTime = True 117 showEventFlops = True 118 arches = sizes.keys() 119 # Time 120 if showTime: 121 data = [] 122 for arch in arches: 123 data.append(sizes[arch]) 124 data.append(times[arch]) 125 plot(*data) 126 title('Performance on '+library+' Example '+str(num)) 127 xlabel('Number of Dof') 128 ylabel('Time (s)') 129 legend(arches, 'upper left', shadow = True) 130 show() 131 # Common event time 132 # We could make a stacked plot like Rio uses here 133 if showEventTime: 134 data = [] 135 names = [] 136 for event, color in [('VecMDot', 'b'), ('VecMAXPY', 'g'), ('MatMult', 'r')]: 137 for arch, style in zip(arches, ['-', ':']): 138 names.append(arch+' '+event) 139 data.append(sizes[arch]) 140 data.append(np.array(events[arch][event])[:,0]) 141 data.append(color+style) 142 plot(*data) 143 title('Performance on '+library+' Example '+str(num)) 144 xlabel('Number of Dof') 145 ylabel('Time (s)') 146 legend(names, 'upper left', shadow = True) 147 show() 148 # Common event flops 149 # We could make a stacked plot like Rio uses here 150 if showEventFlops: 151 data = [] 152 names = [] 153 for event, color in [('VecMDot', 'b'), ('VecMAXPY', 'g'), ('MatMult', 'r')]: 154 for arch, style in zip(arches, ['-', ':']): 155 names.append(arch+' '+event) 156 data.append(sizes[arch]) 157 data.append(np.array(events[arch][event])[:,1]) 158 data.append(color+style) 159 plot(*data) 160 title('Performance on '+library+' Example '+str(num)) 161 xlabel('Number of Dof') 162 ylabel('Computation Rate (MF/s)') 163 legend(names, 'upper left', shadow = True) 164 show() 165 return 166 167def plotSummaryBar(library, num, sizes, times, events): 168 import numpy as np 169 import matplotlib.pyplot as plt 170 171 eventNames = ['VecMDot', 'VecMAXPY', 'MatMult'] 172 eventColors = ['b', 'g', 'r'] 173 arches = sizes.keys() 174 names = [] 175 N = len(sizes[arches[0]]) 176 width = 0.2 177 ind = np.arange(N) - 0.25 178 bars = {} 179 for arch in arches: 180 bars[arch] = [] 181 bottom = np.zeros(N) 182 for event, color in zip(eventNames, eventColors): 183 names.append(arch+' '+event) 184 times = np.array(events[arch][event])[:,0] 185 bars[arch].append(plt.bar(ind, times, width, color=color, bottom=bottom)) 186 bottom += times 187 ind += 0.3 188 189 plt.xlabel('Number of Dof') 190 plt.ylabel('Time (s)') 191 plt.title('GPU vs. CPU Performance on '+library+' Example '+str(num)) 192 plt.xticks(np.arange(N), map(str, sizes[arches[0]])) 193 #plt.yticks(np.arange(0,81,10)) 194 #plt.legend( (p1[0], p2[0]), ('Men', 'Women') ) 195 plt.legend([bar[0] for bar in bars[arches[0]]], eventNames, 'upper right', shadow = True) 196 197 plt.show() 198 return 199 200if __name__ == '__main__': 201 import argparse 202 203 parser = argparse.ArgumentParser(description = 'PETSc Benchmarking', 204 epilog = 'This script runs src/<library>/examples/tutorials/ex<num>, For more information, visit http://www.mcs.anl.gov/petsc', 205 formatter_class = argparse.ArgumentDefaultsHelpFormatter) 206 parser.add_argument('--library', default='SNES', help='The PETSc library used in this example') 207 parser.add_argument('--num', type = int, default='5', help='The example number') 208 parser.add_argument('--module', default='summary', help='The module for timing output') 209 parser.add_argument('--scaling', help='Run parallel scaling test') 210 parser.add_argument('--size', nargs='+', default=['10'], help='Grid size (implementation dependent)') 211 parser.add_argument('--comp', type = int, default='1', help='Number of field components') 212 parser.add_argument('runs', nargs='*', help='Run descriptions: <name>=<args>') 213 parser.add_argument('--stage', default='Main_Stage', help='The default logging stage') 214 parser.add_argument('--events', nargs='+', help='Events to process') 215 parser.add_argument('--batch', action='store_true', default=False, help='Generate batch files for the runs instead') 216 217 # Options for ex19: pc_type='none', dmmg_nlevels=1, mat_no_inode=None 218 219 args = parser.parse_args() 220 print(args) 221 ex = PETScExample(args.library, args.num, log_summary='summary.dat', log_summary_python = None if args.batch else args.module+'.py', preload='off') 222 sizes = {} 223 times = {} 224 events = {} 225 for run in args.runs: 226 name, stropts = run.split('=', 1) 227 opts = dict([t if len(t) == 2 else (t[0], None) for t in [arg.split('=', 1) for arg in stropts.split(' ')]]) 228 sizes[name] = [] 229 times[name] = [] 230 events[name] = {} 231 # DMDA implementation 232 # Need a good way to get the DMDA info 233 for n in map(int, args.size): 234 print(name,opts) 235 ex.run(da_grid_x=n, da_grid_y=n, **opts) 236 sizes[name].append(n*n * args.comp) 237 processSummary('summary', args.stage, args.events, times[name], events[name]) 238 print sizes 239 print times 240 print events 241 if not args.batch: plotSummaryLine(args.library, args.num, sizes, times, events) 242# --num 19 --comp 4 --size 10 20 50 100 --events VecMDot VecMAXPY KSPGMRESOrthog MatMult VecCUSPCopyTo VecCUSPCopyFrom MatCUSPCopyTo --stage Solve CPU='da_vec_type=seq da_mat_type=seqaij' GPU='da_vec_type=seqcusp da_mat_type=seqaijcusp cusp_synchronize' 243