1#!/usr/bin/env python 2import os,sys 3from benchmarkBatch import generateBatchScript 4 5class PETSc(object): 6 def __init__(self): 7 return 8 9 def dir(self): 10 '''Return the root directory for the PETSc tree (usually $PETSC_DIR)''' 11 # This should search for a valid PETSc 12 return os.environ['PETSC_DIR'] 13 14 def arch(self): 15 '''Return the PETSc build label (usually $PETSC_ARCH)''' 16 # This should be configurable 17 return os.environ['PETSC_ARCH'] 18 19 def mpiexec(self): 20 '''Return the path for the mpi launch executable''' 21 mpiexec = os.path.join(self.dir(), self.arch(), 'bin', 'mpiexec') 22 if not os.path.isfile(mpiexec): 23 return None 24 return mpiexec 25 26 def example(self, num): 27 '''Return the path to the executable for a given example number''' 28 return os.path.join(self.dir(), self.arch(), 'lib', 'ex'+str(num)+'-obj', 'ex'+str(num)) 29 30class PETScExample(object): 31 def __init__(self, library, num, **defaultOptions): 32 self.petsc = PETSc() 33 self.library = library 34 self.num = num 35 self.opts = defaultOptions 36 return 37 38 @staticmethod 39 def runShellCommand(command, cwd = None): 40 import subprocess 41 42 Popen = subprocess.Popen 43 PIPE = subprocess.PIPE 44 print 'Executing: %s\n' % (command,) 45 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, bufsize=-1, shell=True, universal_newlines=True) 46 (out, err) = pipe.communicate() 47 ret = pipe.returncode 48 return (out, err, ret) 49 50 def optionsToString(self, **opts): 51 '''Convert a dictionary of options to a command line argument string''' 52 a = [] 53 for key,value in opts.iteritems(): 54 if value is None: 55 a.append('-'+key) 56 else: 57 a.append('-'+key+' '+str(value)) 58 return ' '.join(a) 59 60 def run(self, numProcs = 1, **opts): 61 if self.petsc.mpiexec() is None: 62 cmd = self.petsc.example(self.num) 63 else: 64 cmd = ' '.join([self.petsc.mpiexec(), '-n', str(numProcs), self.petsc.example(self.num)]) 65 cmd += ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts) 66 if 'batch' in opts and opts['batch']: 67 del opts['batch'] 68 filename = generateBatchScript(self.num, numProcs, 120, ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts)) 69 # Submit job 70 out, err, ret = self.runShellCommand('qsub -q gpu '+filename) 71 if ret: 72 print err 73 print out 74 else: 75 out, err, ret = self.runShellCommand(cmd) 76 if ret: 77 print err 78 print out 79 return 80 81def processSummary(moduleName, defaultStage, eventNames, times, events): 82 '''Process the Python log summary into plot data''' 83 m = __import__(moduleName) 84 reload(m) 85 # Total Time 86 times.append(m.Time[0]) 87 # Particular events 88 for name in eventNames: 89 if name.find(':') >= 0: 90 stageName, name = name.split(':', 1) 91 stage = getattr(m, stageName) 92 else: 93 stage = getattr(m, defaultStage) 94 if name in stage.event: 95 if not name in events: 96 events[name] = [] 97 events[name].append((stage.event[name].Time[0], stage.event[name].Flops[0]/(stage.event[name].Time[0] * 1e6))) 98 return 99 100def plotSummaryLine(library, num, sizes, times, events): 101 from pylab import legend, plot, show, title, xlabel, ylabel 102 import numpy as np 103 showTime = False 104 showEventTime = True 105 showEventFlops = True 106 arches = sizes.keys() 107 # Time 108 if showTime: 109 data = [] 110 for arch in arches: 111 data.append(sizes[arch]) 112 data.append(times[arch]) 113 plot(*data) 114 title('Performance on '+library+' Example '+str(num)) 115 xlabel('Number of Dof') 116 ylabel('Time (s)') 117 legend(arches, 'upper left', shadow = True) 118 show() 119 # Common event time 120 # We could make a stacked plot like Rio uses here 121 if showEventTime: 122 data = [] 123 names = [] 124 for event, color in [('VecMDot', 'b'), ('VecMAXPY', 'g'), ('MatMult', 'r')]: 125 for arch, style in zip(arches, ['-', ':']): 126 names.append(arch+' '+event) 127 data.append(sizes[arch]) 128 data.append(np.array(events[arch][event])[:,0]) 129 data.append(color+style) 130 plot(*data) 131 title('Performance on '+library+' Example '+str(num)) 132 xlabel('Number of Dof') 133 ylabel('Time (s)') 134 legend(names, 'upper left', shadow = True) 135 show() 136 # Common event flops 137 # We could make a stacked plot like Rio uses here 138 if showEventFlops: 139 data = [] 140 names = [] 141 for event, color in [('VecMDot', 'b'), ('VecMAXPY', 'g'), ('MatMult', 'r')]: 142 for arch, style in zip(arches, ['-', ':']): 143 names.append(arch+' '+event) 144 data.append(sizes[arch]) 145 data.append(np.array(events[arch][event])[:,1]) 146 data.append(color+style) 147 plot(*data) 148 title('Performance on '+library+' Example '+str(num)) 149 xlabel('Number of Dof') 150 ylabel('Computation Rate (MF/s)') 151 legend(names, 'upper left', shadow = True) 152 show() 153 return 154 155def plotSummaryBar(library, num, sizes, times, events): 156 import numpy as np 157 import matplotlib.pyplot as plt 158 159 eventNames = ['VecMDot', 'VecMAXPY', 'MatMult'] 160 eventColors = ['b', 'g', 'r'] 161 arches = sizes.keys() 162 names = [] 163 N = len(sizes[arches[0]]) 164 width = 0.2 165 ind = np.arange(N) - 0.25 166 bars = {} 167 for arch in arches: 168 bars[arch] = [] 169 bottom = np.zeros(N) 170 for event, color in zip(eventNames, eventColors): 171 names.append(arch+' '+event) 172 times = np.array(events[arch][event])[:,0] 173 bars[arch].append(plt.bar(ind, times, width, color=color, bottom=bottom)) 174 bottom += times 175 ind += 0.3 176 177 plt.xlabel('Number of Dof') 178 plt.ylabel('Time (s)') 179 plt.title('GPU vs. CPU Performance on '+library+' Example '+str(num)) 180 plt.xticks(np.arange(N), map(str, sizes[arches[0]])) 181 #plt.yticks(np.arange(0,81,10)) 182 #plt.legend( (p1[0], p2[0]), ('Men', 'Women') ) 183 plt.legend([bar[0] for bar in bars[arches[0]]], eventNames, 'upper right', shadow = True) 184 185 plt.show() 186 return 187 188if __name__ == '__main__': 189 import argparse 190 191 parser = argparse.ArgumentParser(description = 'PETSc Benchmarking', 192 epilog = 'This script runs src/<library>/examples/tutorials/ex<num>, For more information, visit http://www.mcs.anl.gov/petsc', 193 formatter_class = argparse.ArgumentDefaultsHelpFormatter) 194 parser.add_argument('--library', default='SNES', help='The PETSc library used in this example') 195 parser.add_argument('--num', type = int, default='5', help='The example number') 196 parser.add_argument('--module', default='summary', help='The module for timing output') 197 parser.add_argument('--scaling', help='Run parallel scaling test') 198 parser.add_argument('--size', nargs='+', default=['10'], help='Grid size (implementation dependent)') 199 parser.add_argument('--comp', type = int, default='1', help='Number of field components') 200 parser.add_argument('runs', nargs='*', help='Run descriptions: <name>=<args>') 201 parser.add_argument('--stage', default='Main_Stage', help='The default logging stage') 202 parser.add_argument('--events', nargs='+', help='Events to process') 203 parser.add_argument('--batch', action='store_true', default=False, help='Generate batch files for the runs instead') 204 205 # Options for ex19: pc_type='none', dmmg_nlevels=1, mat_no_inode=None 206 207 args = parser.parse_args() 208 print(args) 209 ex = PETScExample(args.library, args.num, log_summary='summary.dat', log_summary_python = None if args.batch else args.module+'.py', preload='off') 210 sizes = {} 211 times = {} 212 events = {} 213 for run in args.runs: 214 name, stropts = run.split('=', 1) 215 opts = dict([t if len(t) == 2 else (t[0], None) for t in [arg.split('=', 1) for arg in stropts.split(' ')]]) 216 sizes[name] = [] 217 times[name] = [] 218 events[name] = {} 219 # DMDA implementation 220 # Need a good way to get the DMDA info 221 for n in map(int, args.size): 222 print(name,opts) 223 ex.run(da_grid_x=n, da_grid_y=n, **opts) 224 sizes[name].append(n*n * args.comp) 225 processSummary('summary', args.stage, args.events, times[name], events[name]) 226 print sizes 227 print times 228 print events 229 if not args.batch: plotSummaryLine(args.library, args.num, sizes, times, events) 230# --num 19 --comp 4 --size 10 20 50 100 --events VecMDot VecMAXPY KSPGMRESOrthog MatMult VecCUSPCopyTo VecCUSPCopyFrom MatCUSPCopyTo --stage Solve CPU='da_vec_type=seq da_mat_type=seqaij' GPU='da_vec_type=seqcusp da_mat_type=seqaijcusp cusp_synchronize' 231