13428b40fSMatthew G Knepley#!/usr/bin/env python 25b6bfdb9SJed Brownfrom __future__ import print_function 3eda8839fSMatthew G Knepleyimport os,sys 4683aebbfSMatthew G Knepleysys.path.append(os.path.join(os.environ['PETSC_DIR'], 'config')) 590dfb094SAndy R. Terrelsys.path.append(os.getcwd()) 62247410dSMatthew G. Knepley 72247410dSMatthew G. Knepleyclass CSVLog(object): pass 83428b40fSMatthew G Knepley 93428b40fSMatthew G Knepleyclass PETSc(object): 103428b40fSMatthew G Knepley def __init__(self): 113428b40fSMatthew G Knepley return 123428b40fSMatthew G Knepley 133428b40fSMatthew G Knepley def dir(self): 143428b40fSMatthew G Knepley '''Return the root directory for the PETSc tree (usually $PETSC_DIR)''' 153428b40fSMatthew G Knepley # This should search for a valid PETSc 163428b40fSMatthew G Knepley return os.environ['PETSC_DIR'] 173428b40fSMatthew G Knepley 183428b40fSMatthew G Knepley def arch(self): 193428b40fSMatthew G Knepley '''Return the PETSc build label (usually $PETSC_ARCH)''' 203428b40fSMatthew G Knepley # This should be configurable 213428b40fSMatthew G Knepley return os.environ['PETSC_ARCH'] 223428b40fSMatthew G Knepley 233428b40fSMatthew G Knepley def mpiexec(self): 243428b40fSMatthew G Knepley '''Return the path for the mpi launch executable''' 25e3da8a91SMatthew G Knepley mpiexec = os.path.join(self.dir(), self.arch(), 'bin', 'mpiexec') 266cbfa02cSMatthew G Knepley if not os.path.isfile(mpiexec): 27e3da8a91SMatthew G Knepley return None 28e3da8a91SMatthew G Knepley return mpiexec 293428b40fSMatthew G Knepley 30d37947eaSMatthew G. Knepley def example(self, library, num): 313428b40fSMatthew G Knepley '''Return the path to the executable for a given example number''' 32d37947eaSMatthew G. Knepley return os.path.join(self.dir(), self.arch(), 'lib', library.lower(), 'ex'+str(num)) 333428b40fSMatthew G Knepley 343973d8daSKarl Rupp def source(self, library, num, filenametail): 350790b1abSMatthew G Knepley '''Return the path to the sources for a given example number''' 360790b1abSMatthew G Knepley d = os.path.join(self.dir(), 'src', library.lower(), 'examples', 'tutorials') 370790b1abSMatthew G Knepley name = 'ex'+str(num) 380790b1abSMatthew G Knepley sources = [] 390790b1abSMatthew G Knepley for f in os.listdir(d): 400790b1abSMatthew G Knepley if f == name+'.c': 412642ea08SMatthew G Knepley sources.insert(0, f) 423973d8daSKarl Rupp elif f.startswith(name) and f.endswith(filenametail): 430790b1abSMatthew G Knepley sources.append(f) 440790b1abSMatthew G Knepley return map(lambda f: os.path.join(d, f), sources) 450790b1abSMatthew G Knepley 463428b40fSMatthew G Knepleyclass PETScExample(object): 473428b40fSMatthew G Knepley def __init__(self, library, num, **defaultOptions): 483428b40fSMatthew G Knepley self.petsc = PETSc() 493428b40fSMatthew G Knepley self.library = library 503428b40fSMatthew G Knepley self.num = num 513428b40fSMatthew G Knepley self.opts = defaultOptions 523428b40fSMatthew G Knepley return 533428b40fSMatthew G Knepley 543428b40fSMatthew G Knepley @staticmethod 55d006b754SMatthew G Knepley def runShellCommand(command, cwd = None, log = True): 563428b40fSMatthew G Knepley import subprocess 573428b40fSMatthew G Knepley 583428b40fSMatthew G Knepley Popen = subprocess.Popen 593428b40fSMatthew G Knepley PIPE = subprocess.PIPE 605b6bfdb9SJed Brown if log: print('Executing: %s\n' % (command,)) 613428b40fSMatthew G Knepley pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, bufsize=-1, shell=True, universal_newlines=True) 623428b40fSMatthew G Knepley (out, err) = pipe.communicate() 633428b40fSMatthew G Knepley ret = pipe.returncode 643428b40fSMatthew G Knepley return (out, err, ret) 653428b40fSMatthew G Knepley 663428b40fSMatthew G Knepley def optionsToString(self, **opts): 673428b40fSMatthew G Knepley '''Convert a dictionary of options to a command line argument string''' 683428b40fSMatthew G Knepley a = [] 69a01632ecSJed Brown for key,value in opts.items(): 703428b40fSMatthew G Knepley if value is None: 713428b40fSMatthew G Knepley a.append('-'+key) 723428b40fSMatthew G Knepley else: 733428b40fSMatthew G Knepley a.append('-'+key+' '+str(value)) 743428b40fSMatthew G Knepley return ' '.join(a) 753428b40fSMatthew G Knepley 76d37947eaSMatthew G. Knepley def build(self, log = True): 77d37947eaSMatthew G. Knepley sdir = os.path.join(self.petsc.dir(), 'src', self.library.lower(), 'examples', 'tutorials') 78d37947eaSMatthew G. Knepley odir = os.getcwd() 79d37947eaSMatthew G. Knepley os.chdir(sdir) 80d37947eaSMatthew G. Knepley cmd = 'make ex'+str(self.num) 81d37947eaSMatthew G. Knepley out, err, ret = self.runShellCommand(cmd, cwd = sdir, log = log) 82d37947eaSMatthew G. Knepley if err: 83d37947eaSMatthew G. Knepley raise RuntimeError('Unable to build example:\n'+err+out) 84d37947eaSMatthew G. Knepley os.chdir(odir) 85d37947eaSMatthew G. Knepley bdir = os.path.dirname(self.petsc.example(self.library, self.num)) 86d37947eaSMatthew G. Knepley try: 87d37947eaSMatthew G. Knepley os.makedirs(bdir) 88d37947eaSMatthew G. Knepley except OSError: 89d37947eaSMatthew G. Knepley if not os.path.isdir(bdir): 90d37947eaSMatthew G. Knepley raise 91d37947eaSMatthew G. Knepley cmd = 'mv '+os.path.join(sdir, 'ex'+str(self.num))+' '+self.petsc.example(self.library, self.num) 92d37947eaSMatthew G. Knepley out, err, ret = self.runShellCommand(cmd, log = log) 93d37947eaSMatthew G. Knepley if ret: 945b6bfdb9SJed Brown print(err) 955b6bfdb9SJed Brown print(out) 96d37947eaSMatthew G. Knepley return 97d37947eaSMatthew G. Knepley 98d006b754SMatthew G Knepley def run(self, numProcs = 1, log = True, **opts): 99b3efd4cdSAndy R. Terrel cmd = '' 100b3efd4cdSAndy R. Terrel if self.petsc.mpiexec() is not None: 101b3efd4cdSAndy R. Terrel cmd += self.petsc.mpiexec() + ' ' 102b3efd4cdSAndy R. Terrel numProcs = os.environ.get('NUM_RANKS', numProcs) 103b3efd4cdSAndy R. Terrel cmd += ' -n ' + str(numProcs) + ' ' 1045b6bfdb9SJed Brown if 'PE_HOSTFILE' in os.environ: 105b3efd4cdSAndy R. Terrel cmd += ' -hostfile hostfile ' 106d37947eaSMatthew G. Knepley cmd += ' '.join([self.petsc.example(self.library, self.num), self.optionsToString(**self.opts), self.optionsToString(**opts)]) 10719d5f70aSMatthew G Knepley if 'batch' in opts and opts['batch']: 10819d5f70aSMatthew G Knepley del opts['batch'] 1092247410dSMatthew G. Knepley from benchmarkBatch import generateBatchScript 1103849a283SMatthew G Knepley filename = generateBatchScript(self.num, numProcs, 120, ' '+self.optionsToString(**self.opts)+' '+self.optionsToString(**opts)) 1113849a283SMatthew G Knepley # Submit job 112d006b754SMatthew G Knepley out, err, ret = self.runShellCommand('qsub -q gpu '+filename, log = log) 1133849a283SMatthew G Knepley if ret: 1145b6bfdb9SJed Brown print(err) 1155b6bfdb9SJed Brown print(out) 11619d5f70aSMatthew G Knepley else: 117d006b754SMatthew G Knepley out, err, ret = self.runShellCommand(cmd, log = log) 1183428b40fSMatthew G Knepley if ret: 1195b6bfdb9SJed Brown print(err) 1205b6bfdb9SJed Brown print(out) 1210790b1abSMatthew G Knepley return out 1223428b40fSMatthew G Knepley 1232247410dSMatthew G. Knepleydef processSummaryCSV(filename, defaultStage, eventNames, sizes, times, errors, events): 1242247410dSMatthew G. Knepley '''Process the CSV log summary into plot data''' 1252247410dSMatthew G. Knepley import csv 1262247410dSMatthew G. Knepley m = CSVLog() 1272247410dSMatthew G. Knepley setattr(m, 'Stages', {}) 1282247410dSMatthew G. Knepley with open(filename) as csvfile: 1292247410dSMatthew G. Knepley reader = csv.DictReader(csvfile) 1302247410dSMatthew G. Knepley for row in reader: 1312247410dSMatthew G. Knepley stageName = row["Stage Name"] 1322247410dSMatthew G. Knepley eventName = row["Event Name"] 1332247410dSMatthew G. Knepley rank = int(row["Rank"]) 1342247410dSMatthew G. Knepley if not stageName in m.Stages: m.Stages[stageName] = {} 1352247410dSMatthew G. Knepley if not eventName in m.Stages[stageName]: m.Stages[stageName][eventName] = {} 1362247410dSMatthew G. Knepley m.Stages[stageName][eventName][rank] = {"time" : float(row["Time"]), "numMessages": float(row["Num Messages"]), "messageLength": float(row["Message Length"]), "numReductions" : float(row["Num Reductions"]), "flop" : float(row["FLOP"])} 1372247410dSMatthew G. Knepley for i in range(8): 1382247410dSMatthew G. Knepley dname = "dof"+str(i) 1392247410dSMatthew G. Knepley ename = "e"+str(i) 1402247410dSMatthew G. Knepley if row[dname]: 1412247410dSMatthew G. Knepley if not "dof" in m.Stages[stageName][eventName][rank]: 1422247410dSMatthew G. Knepley m.Stages[stageName][eventName][rank]["dof"] = [] 1432247410dSMatthew G. Knepley m.Stages[stageName][eventName][rank]["error"] = [] 1442247410dSMatthew G. Knepley m.Stages[stageName][eventName][rank]["dof"].append(int(float(row[dname]))) 1452247410dSMatthew G. Knepley m.Stages[stageName][eventName][rank]["error"].append(float(row[ename])) 1462247410dSMatthew G. Knepley return m 1472247410dSMatthew G. Knepley 1482247410dSMatthew G. Knepleydef processSummary(moduleName, defaultStage, eventNames, sizes, times, errors, events): 1493428b40fSMatthew G Knepley '''Process the Python log summary into plot data''' 1502247410dSMatthew G. Knepley if os.path.isfile(moduleName+'.csv'): 1512247410dSMatthew G. Knepley m = processSummaryCSV(moduleName+'.csv', defaultStage, eventNames, sizes, times, errors, events) 1522247410dSMatthew G. Knepley else: 1533428b40fSMatthew G Knepley m = __import__(moduleName) 1543428b40fSMatthew G Knepley # Total Time 1552247410dSMatthew G. Knepley times.append(m.Stages[defaultStage]["summary"][0]["time"]) 1563428b40fSMatthew G Knepley # Particular events 157eda8839fSMatthew G Knepley for name in eventNames: 158eda8839fSMatthew G Knepley if name.find(':') >= 0: 159eda8839fSMatthew G Knepley stageName, name = name.split(':', 1) 160eda8839fSMatthew G Knepley else: 1612247410dSMatthew G. Knepley stageName = defaultStage 1622247410dSMatthew G. Knepley stage = m.Stages[stageName] 1632247410dSMatthew G. Knepley if name in stage: 1643428b40fSMatthew G Knepley if not name in events: 1653428b40fSMatthew G Knepley events[name] = [] 1662247410dSMatthew G. Knepley event = stage[name][0] 1672247410dSMatthew G. Knepley etimes = [stage[name][p]["time"] for p in stage[name]] 1682247410dSMatthew G. Knepley eflops = [stage[name][p]["flop"] for p in stage[name]] 1692247410dSMatthew G. Knepley if "dof" in event: 1702247410dSMatthew G. Knepley sizes.append(event["dof"][0]) 1712247410dSMatthew G. Knepley errors.append(event["error"][0]) 172df494a56SMatthew G Knepley try: 1732247410dSMatthew G. Knepley events[name].append((max(etimes), sum(eflops)/(max(etimes) * 1e6))) 174df494a56SMatthew G Knepley except ZeroDivisionError: 1752247410dSMatthew G. Knepley events[name].append((max(etimes), 0)) 1763428b40fSMatthew G Knepley return 1773428b40fSMatthew G Knepley 1786e25a272SMatthew G Knepleydef plotTime(library, num, eventNames, sizes, times, events): 1796e25a272SMatthew G Knepley from pylab import legend, plot, show, title, xlabel, ylabel 1806e25a272SMatthew G Knepley import numpy as np 1816e25a272SMatthew G Knepley 1826e25a272SMatthew G Knepley arches = sizes.keys() 1836e25a272SMatthew G Knepley data = [] 1846e25a272SMatthew G Knepley for arch in arches: 1856e25a272SMatthew G Knepley data.append(sizes[arch]) 1866e25a272SMatthew G Knepley data.append(times[arch]) 1876e25a272SMatthew G Knepley plot(*data) 1886e25a272SMatthew G Knepley title('Performance on '+library+' Example '+str(num)) 1896e25a272SMatthew G Knepley xlabel('Number of Dof') 1906e25a272SMatthew G Knepley ylabel('Time (s)') 1916e25a272SMatthew G Knepley legend(arches, 'upper left', shadow = True) 1926e25a272SMatthew G Knepley show() 1936e25a272SMatthew G Knepley return 1946e25a272SMatthew G Knepley 1956e25a272SMatthew G Knepleydef plotEventTime(library, num, eventNames, sizes, times, events, filename = None): 1966e25a272SMatthew G Knepley from pylab import close, legend, plot, savefig, show, title, xlabel, ylabel 1976e25a272SMatthew G Knepley import numpy as np 1986e25a272SMatthew G Knepley 1996e25a272SMatthew G Knepley close() 2006e25a272SMatthew G Knepley arches = sizes.keys() 2016e25a272SMatthew G Knepley bs = events[arches[0]].keys()[0] 2026e25a272SMatthew G Knepley data = [] 2036e25a272SMatthew G Knepley names = [] 2046e25a272SMatthew G Knepley for event, color in zip(eventNames, ['b', 'g', 'r', 'y']): 2056e25a272SMatthew G Knepley for arch, style in zip(arches, ['-', ':']): 2066e25a272SMatthew G Knepley if event in events[arch][bs]: 2076e25a272SMatthew G Knepley names.append(arch+'-'+str(bs)+' '+event) 2086e25a272SMatthew G Knepley data.append(sizes[arch][bs]) 2096e25a272SMatthew G Knepley data.append(np.array(events[arch][bs][event])[:,0]) 2106e25a272SMatthew G Knepley data.append(color+style) 2116e25a272SMatthew G Knepley else: 2125b6bfdb9SJed Brown print('Could not find %s in %s-%d events' % (event, arch, bs)) 2135b6bfdb9SJed Brown print(data) 2146e25a272SMatthew G Knepley plot(*data) 2156e25a272SMatthew G Knepley title('Performance on '+library+' Example '+str(num)) 2166e25a272SMatthew G Knepley xlabel('Number of Dof') 2176e25a272SMatthew G Knepley ylabel('Time (s)') 2186e25a272SMatthew G Knepley legend(names, 'upper left', shadow = True) 2196e25a272SMatthew G Knepley if filename is None: 2206e25a272SMatthew G Knepley show() 2216e25a272SMatthew G Knepley else: 2226e25a272SMatthew G Knepley savefig(filename) 2236e25a272SMatthew G Knepley return 2246e25a272SMatthew G Knepley 2256e25a272SMatthew G Knepleydef plotEventFlop(library, num, eventNames, sizes, times, events, filename = None): 2266e25a272SMatthew G Knepley from pylab import legend, plot, savefig, semilogy, show, title, xlabel, ylabel 2276e25a272SMatthew G Knepley import numpy as np 2286e25a272SMatthew G Knepley 2296e25a272SMatthew G Knepley arches = sizes.keys() 2306e25a272SMatthew G Knepley bs = events[arches[0]].keys()[0] 2316e25a272SMatthew G Knepley data = [] 2326e25a272SMatthew G Knepley names = [] 2336e25a272SMatthew G Knepley for event, color in zip(eventNames, ['b', 'g', 'r', 'y']): 2346e25a272SMatthew G Knepley for arch, style in zip(arches, ['-', ':']): 2356e25a272SMatthew G Knepley if event in events[arch][bs]: 2366e25a272SMatthew G Knepley names.append(arch+'-'+str(bs)+' '+event) 2376e25a272SMatthew G Knepley data.append(sizes[arch][bs]) 2386e25a272SMatthew G Knepley data.append(1e-3*np.array(events[arch][bs][event])[:,1]) 2396e25a272SMatthew G Knepley data.append(color+style) 2406e25a272SMatthew G Knepley else: 2415b6bfdb9SJed Brown print('Could not find %s in %s-%d events' % (event, arch, bs)) 2426e25a272SMatthew G Knepley semilogy(*data) 2436e25a272SMatthew G Knepley title('Performance on '+library+' Example '+str(num)) 2446e25a272SMatthew G Knepley xlabel('Number of Dof') 2456e25a272SMatthew G Knepley ylabel('Computation Rate (GF/s)') 2466e25a272SMatthew G Knepley legend(names, 'upper left', shadow = True) 2476e25a272SMatthew G Knepley if filename is None: 2486e25a272SMatthew G Knepley show() 2496e25a272SMatthew G Knepley else: 2506e25a272SMatthew G Knepley savefig(filename) 2516e25a272SMatthew G Knepley return 2526e25a272SMatthew G Knepley 253929aa6beSMatthew G. Knepleydef plotEventScaling(library, num, eventNames, procs, events, filename = None): 254929aa6beSMatthew G. Knepley from pylab import legend, plot, savefig, semilogy, show, title, xlabel, ylabel 255929aa6beSMatthew G. Knepley import numpy as np 256929aa6beSMatthew G. Knepley 257929aa6beSMatthew G. Knepley arches = procs.keys() 258929aa6beSMatthew G. Knepley bs = events[arches[0]].keys()[0] 259929aa6beSMatthew G. Knepley data = [] 260929aa6beSMatthew G. Knepley names = [] 261929aa6beSMatthew G. Knepley for arch, style in zip(arches, ['-', ':']): 262929aa6beSMatthew G. Knepley for event, color in zip(eventNames, ['b', 'g', 'r', 'y']): 263929aa6beSMatthew G. Knepley if event in events[arch][bs]: 264929aa6beSMatthew G. Knepley names.append(arch+'-'+str(bs)+' '+event) 265929aa6beSMatthew G. Knepley data.append(procs[arch][bs]) 266929aa6beSMatthew G. Knepley data.append(1e-3*np.array(events[arch][bs][event])[:,1]) 267929aa6beSMatthew G. Knepley data.append(color+style) 268929aa6beSMatthew G. Knepley else: 2695b6bfdb9SJed Brown print('Could not find %s in %s-%d events' % (event, arch, bs)) 270929aa6beSMatthew G. Knepley plot(*data) 271929aa6beSMatthew G. Knepley title('Performance on '+library+' Example '+str(num)) 272929aa6beSMatthew G. Knepley xlabel('Number of Processors') 273929aa6beSMatthew G. Knepley ylabel('Computation Rate (GF/s)') 274929aa6beSMatthew G. Knepley legend(names, 'upper left', shadow = True) 275929aa6beSMatthew G. Knepley if filename is None: 276929aa6beSMatthew G. Knepley show() 277929aa6beSMatthew G. Knepley else: 278929aa6beSMatthew G. Knepley savefig(filename) 279929aa6beSMatthew G. Knepley return 280929aa6beSMatthew G. Knepley 281303b7b21SMatthew G Knepleydef plotSummaryLine(library, num, eventNames, sizes, times, events): 2823428b40fSMatthew G Knepley from pylab import legend, plot, show, title, xlabel, ylabel 2833428b40fSMatthew G Knepley import numpy as np 2843428b40fSMatthew G Knepley showTime = False 2853428b40fSMatthew G Knepley showEventTime = True 2863428b40fSMatthew G Knepley showEventFlops = True 2873428b40fSMatthew G Knepley arches = sizes.keys() 2883428b40fSMatthew G Knepley # Time 2893428b40fSMatthew G Knepley if showTime: 2903428b40fSMatthew G Knepley data = [] 2913428b40fSMatthew G Knepley for arch in arches: 2923428b40fSMatthew G Knepley data.append(sizes[arch]) 2933428b40fSMatthew G Knepley data.append(times[arch]) 2943428b40fSMatthew G Knepley plot(*data) 2953428b40fSMatthew G Knepley title('Performance on '+library+' Example '+str(num)) 2963428b40fSMatthew G Knepley xlabel('Number of Dof') 2973428b40fSMatthew G Knepley ylabel('Time (s)') 2983428b40fSMatthew G Knepley legend(arches, 'upper left', shadow = True) 2993428b40fSMatthew G Knepley show() 3003428b40fSMatthew G Knepley # Common event time 3013428b40fSMatthew G Knepley # We could make a stacked plot like Rio uses here 3023428b40fSMatthew G Knepley if showEventTime: 3033428b40fSMatthew G Knepley data = [] 3043428b40fSMatthew G Knepley names = [] 305df494a56SMatthew G Knepley for event, color in zip(eventNames, ['b', 'g', 'r', 'y']): 3063428b40fSMatthew G Knepley for arch, style in zip(arches, ['-', ':']): 3072247410dSMatthew G. Knepley if event in events[arch]: 3082247410dSMatthew G. Knepley names.append(arch+' '+event) 3092247410dSMatthew G. Knepley data.append(sizes[arch]) 3102247410dSMatthew G. Knepley data.append(np.array(events[arch][event])[:,0]) 3113428b40fSMatthew G Knepley data.append(color+style) 312df494a56SMatthew G Knepley else: 3132247410dSMatthew G. Knepley print('Could not find %s in %s events' % (event, arch)) 3145b6bfdb9SJed Brown print(data) 3153428b40fSMatthew G Knepley plot(*data) 3163428b40fSMatthew G Knepley title('Performance on '+library+' Example '+str(num)) 3173428b40fSMatthew G Knepley xlabel('Number of Dof') 3183428b40fSMatthew G Knepley ylabel('Time (s)') 3193428b40fSMatthew G Knepley legend(names, 'upper left', shadow = True) 3203428b40fSMatthew G Knepley show() 3213428b40fSMatthew G Knepley # Common event flops 3223428b40fSMatthew G Knepley # We could make a stacked plot like Rio uses here 3233428b40fSMatthew G Knepley if showEventFlops: 3243428b40fSMatthew G Knepley data = [] 3253428b40fSMatthew G Knepley names = [] 326df494a56SMatthew G Knepley for event, color in zip(eventNames, ['b', 'g', 'r', 'y']): 3273428b40fSMatthew G Knepley for arch, style in zip(arches, ['-', ':']): 3282247410dSMatthew G. Knepley if event in events[arch]: 3292247410dSMatthew G. Knepley names.append(arch+' '+event) 3302247410dSMatthew G. Knepley data.append(sizes[arch]) 3312247410dSMatthew G. Knepley data.append(np.array(events[arch][event])[:,1]) 3323428b40fSMatthew G Knepley data.append(color+style) 333df494a56SMatthew G Knepley else: 3342247410dSMatthew G. Knepley print('Could not find %s in %s events' % (event, arch)) 3353428b40fSMatthew G Knepley plot(*data) 3363428b40fSMatthew G Knepley title('Performance on '+library+' Example '+str(num)) 3373428b40fSMatthew G Knepley xlabel('Number of Dof') 3383428b40fSMatthew G Knepley ylabel('Computation Rate (MF/s)') 3393428b40fSMatthew G Knepley legend(names, 'upper left', shadow = True) 3403428b40fSMatthew G Knepley show() 3413428b40fSMatthew G Knepley return 3423428b40fSMatthew G Knepley 3432247410dSMatthew G. Knepleydef plotMeshConvergence(library, num, eventNames, sizes, times, errors, events): 3442247410dSMatthew G. Knepley import numpy as np 3452247410dSMatthew G. Knepley import matplotlib.pyplot as plt 3462247410dSMatthew G. Knepley data = [] 3472247410dSMatthew G. Knepley legends = [] 3482247410dSMatthew G. Knepley print(sizes) 3492247410dSMatthew G. Knepley print(errors) 3502247410dSMatthew G. Knepley for run in sizes: 3512247410dSMatthew G. Knepley rsizes = np.array(sizes[run]) 3522247410dSMatthew G. Knepley data.extend([rsizes, errors[run], rsizes, (errors[run][0]*rsizes[0]*2)*rsizes**(meshExp[run]/-2.0)]) 3532247410dSMatthew G. Knepley legends.extend(['Experiment '+run, r'Synthetic '+run+r' $\alpha = '+str(meshExp[run])+'$']) 3542247410dSMatthew G. Knepley SizeError = plt.loglog(*data) 3552247410dSMatthew G. Knepley plt.title(library+' ex'+str(num)+' Mesh Convergence') 3562247410dSMatthew G. Knepley plt.xlabel('Size') 3572247410dSMatthew G. Knepley plt.ylabel(r'Error $\|x - x^*\|_2$') 3582247410dSMatthew G. Knepley plt.legend(legends) 3592247410dSMatthew G. Knepley plt.show() 3602247410dSMatthew G. Knepley return 3612247410dSMatthew G. Knepley 3622247410dSMatthew G. Knepleydef plotWorkPrecision(library, num, eventNames, sizes, times, errors, events): 3632247410dSMatthew G. Knepley import numpy as np 3642247410dSMatthew G. Knepley import matplotlib.pyplot as plt 3652247410dSMatthew G. Knepley data = [] 3662247410dSMatthew G. Knepley legends = [] 3672247410dSMatthew G. Knepley for run in times: 3682247410dSMatthew G. Knepley rtimes = np.array(times[run]) 3692247410dSMatthew G. Knepley data.extend([rtimes, errors[run], rtimes, (errors[run][0]*rtimes[0]*2)*rtimes**(timeExp[run])]) 3702247410dSMatthew G. Knepley legends.extend(['Experiment '+run, 'Synthetic '+run+' exponent '+str(timeExp[run])]) 3712247410dSMatthew G. Knepley TimeError = plt.loglog(*data) 3722247410dSMatthew G. Knepley plt.title(library+' ex'+str(num)+' Work Precision') 3732247410dSMatthew G. Knepley plt.xlabel('Time (s)') 3742247410dSMatthew G. Knepley plt.ylabel(r'Error $\|x - x^*\|_2$') 3752247410dSMatthew G. Knepley plt.legend(legends) 3762247410dSMatthew G. Knepley plt.show() 3772247410dSMatthew G. Knepley return 3782247410dSMatthew G. Knepley 3792247410dSMatthew G. Knepleydef plotWorkPrecisionPareto(library, num, eventNames, sizes, times, errors, events): 3802247410dSMatthew G. Knepley import numpy as np 3812247410dSMatthew G. Knepley import matplotlib.pyplot as plt 3822247410dSMatthew G. Knepley data = [] 3832247410dSMatthew G. Knepley legends = [] 3842247410dSMatthew G. Knepley for run in times: 3852247410dSMatthew G. Knepley rtimes = np.array(times[run]) 3862247410dSMatthew G. Knepley data.extend([rtimes, errors[run]]) 3872247410dSMatthew G. Knepley legends.append('Experiment '+run) 3882247410dSMatthew G. Knepley TimeErrorPareto = plt.semilogy(*data) 3892247410dSMatthew G. Knepley plt.title(library+' ex'+str(num)+' Work Precision: Pareto Front') 3902247410dSMatthew G. Knepley plt.xlabel('Time (s)') 3912247410dSMatthew G. Knepley plt.ylabel(r'Error $\|x - x^*\|_2$') 3922247410dSMatthew G. Knepley plt.legend(legends) 3932247410dSMatthew G. Knepley plt.show() 3942247410dSMatthew G. Knepley return 3952247410dSMatthew G. Knepley 396303b7b21SMatthew G Knepleydef plotSummaryBar(library, num, eventNames, sizes, times, events): 397e3da8a91SMatthew G Knepley import numpy as np 398e3da8a91SMatthew G Knepley import matplotlib.pyplot as plt 399e3da8a91SMatthew G Knepley 400303b7b21SMatthew G Knepley eventColors = ['b', 'g', 'r', 'y'] 401e3da8a91SMatthew G Knepley arches = sizes.keys() 402e3da8a91SMatthew G Knepley names = [] 403e3da8a91SMatthew G Knepley N = len(sizes[arches[0]]) 404e3da8a91SMatthew G Knepley width = 0.2 405e3da8a91SMatthew G Knepley ind = np.arange(N) - 0.25 406e3da8a91SMatthew G Knepley bars = {} 407e3da8a91SMatthew G Knepley for arch in arches: 408e3da8a91SMatthew G Knepley bars[arch] = [] 409e3da8a91SMatthew G Knepley bottom = np.zeros(N) 410e3da8a91SMatthew G Knepley for event, color in zip(eventNames, eventColors): 411e3da8a91SMatthew G Knepley names.append(arch+' '+event) 412e3da8a91SMatthew G Knepley times = np.array(events[arch][event])[:,0] 413e3da8a91SMatthew G Knepley bars[arch].append(plt.bar(ind, times, width, color=color, bottom=bottom)) 414e3da8a91SMatthew G Knepley bottom += times 415e3da8a91SMatthew G Knepley ind += 0.3 416e3da8a91SMatthew G Knepley 417e3da8a91SMatthew G Knepley plt.xlabel('Number of Dof') 418e3da8a91SMatthew G Knepley plt.ylabel('Time (s)') 419e3da8a91SMatthew G Knepley plt.title('GPU vs. CPU Performance on '+library+' Example '+str(num)) 420e3da8a91SMatthew G Knepley plt.xticks(np.arange(N), map(str, sizes[arches[0]])) 421e3da8a91SMatthew G Knepley #plt.yticks(np.arange(0,81,10)) 422e3da8a91SMatthew G Knepley #plt.legend( (p1[0], p2[0]), ('Men', 'Women') ) 423e3da8a91SMatthew G Knepley plt.legend([bar[0] for bar in bars[arches[0]]], eventNames, 'upper right', shadow = True) 424e3da8a91SMatthew G Knepley 425e3da8a91SMatthew G Knepley plt.show() 426e3da8a91SMatthew G Knepley return 427e3da8a91SMatthew G Knepley 4282247410dSMatthew G. Knepleydef processOptions(opts, name, n): 4292247410dSMatthew G. Knepley newopts = {} 4302247410dSMatthew G. Knepley for key, val in opts.items(): 4312247410dSMatthew G. Knepley val = opts[key] 4322247410dSMatthew G. Knepley if val and val.find('%') >= 0: 4332247410dSMatthew G. Knepley newval = val % (name.replace('/', '-'), n) 4342247410dSMatthew G. Knepley newopts[key] = newval 4352247410dSMatthew G. Knepley else: 4362247410dSMatthew G. Knepley newopts[key] = val 4372247410dSMatthew G. Knepley return newopts 4382247410dSMatthew G. Knepley 4392247410dSMatthew G. Knepleydef getLogName(opts): 4402247410dSMatthew G. Knepley if 'log_view' in opts: 4412247410dSMatthew G. Knepley val = opts['log_view'] 4422247410dSMatthew G. Knepley s = val.find(':') 4432247410dSMatthew G. Knepley e = val.find(':', s+1) 4442247410dSMatthew G. Knepley logName = os.path.splitext(val[s+1:e])[0] 4452247410dSMatthew G. Knepley return logName 4462247410dSMatthew G. Knepley return None 447683aebbfSMatthew G Knepley 4484992a29cSMatthew G. Knepleydef run_DMDA(ex, name, opts, args, sizes, times, events, log=True, execute=True): 449683aebbfSMatthew G Knepley for n in map(int, args.size): 4502247410dSMatthew G. Knepley newopts = processOptions(opts, name, n) 4514992a29cSMatthew G. Knepley if execute: 4522247410dSMatthew G. Knepley ex.run(log=log, da_grid_x=n, da_grid_y=n, **newopts) 4532247410dSMatthew G. Knepley processSummary(getLogName(newopts), args.stage, args.events, sizes[name], times[name], errors[name], events[name]) 454683aebbfSMatthew G Knepley return 455683aebbfSMatthew G Knepley 4564992a29cSMatthew G. Knepleydef run_DMPlex(ex, name, opts, args, sizes, times, events, log=True, execute=True): 4572247410dSMatthew G. Knepley newopts = processOptions(opts, name, args.refine) 4584992a29cSMatthew G. Knepley if execute: 4592247410dSMatthew G. Knepley ex.run(log=log, dim=args.dim, snes_convergence_estimate=None, convest_num_refine=args.refine, interpolate=1, **newopts) 4602247410dSMatthew G. Knepley for r in range(args.refine+1): 4612247410dSMatthew G. Knepley stage = args.stage 4622247410dSMatthew G. Knepley if stage.find('%') >= 0: stage = stage % (r) 4632247410dSMatthew G. Knepley processSummary(getLogName(newopts), stage, args.events, sizes[name], times[name], errors[name], events[name]) 464683aebbfSMatthew G Knepley return 465683aebbfSMatthew G Knepley 466d006b754SMatthew G Knepleydef outputData(sizes, times, events, name = 'output.py'): 467d006b754SMatthew G Knepley if os.path.exists(name): 468d006b754SMatthew G Knepley base, ext = os.path.splitext(name) 469d006b754SMatthew G Knepley num = 1 470d006b754SMatthew G Knepley while os.path.exists(base+str(num)+ext): 471d006b754SMatthew G Knepley num += 1 472d006b754SMatthew G Knepley name = base+str(num)+ext 473d006b754SMatthew G Knepley with file(name, 'w') as f: 474d006b754SMatthew G Knepley f.write('#PETSC_ARCH='+os.environ['PETSC_ARCH']+' '+' '.join(sys.argv)+'\n') 475d006b754SMatthew G Knepley f.write('sizes = '+repr(sizes)+'\n') 476d006b754SMatthew G Knepley f.write('times = '+repr(times)+'\n') 477d006b754SMatthew G Knepley f.write('events = '+repr(events)+'\n') 478d006b754SMatthew G Knepley return 479d006b754SMatthew G Knepley 4803428b40fSMatthew G Knepleyif __name__ == '__main__': 481eda8839fSMatthew G Knepley import argparse 4822247410dSMatthew G. Knepley import __main__ 483eda8839fSMatthew G Knepley 484eda8839fSMatthew G Knepley parser = argparse.ArgumentParser(description = 'PETSc Benchmarking', 485*a17b96a8SKyle Gerard Felker epilog = 'This script runs src/<library>/tutorials/ex<num>, For more information, visit https://petsc.org/', 486eda8839fSMatthew G Knepley formatter_class = argparse.ArgumentDefaultsHelpFormatter) 487eda8839fSMatthew G Knepley parser.add_argument('--library', default='SNES', help='The PETSc library used in this example') 488eda8839fSMatthew G Knepley parser.add_argument('--num', type = int, default='5', help='The example number') 489eda8839fSMatthew G Knepley parser.add_argument('--module', default='summary', help='The module for timing output') 4902247410dSMatthew G. Knepley parser.add_argument('--stage', default='Main Stage', help='The default logging stage') 491eda8839fSMatthew G Knepley parser.add_argument('--events', nargs='+', help='Events to process') 4924992a29cSMatthew G. Knepley parser.add_argument('--plotOnly',action='store_true', default=False, help='Flag to only plot existing data') 493eda8839fSMatthew G Knepley parser.add_argument('--batch', action='store_true', default=False, help='Generate batch files for the runs instead') 494d006b754SMatthew G Knepley parser.add_argument('--daemon', action='store_true', default=False, help='Run as a daemon') 4953973d8daSKarl Rupp parser.add_argument('--gpulang', default='OpenCL', help='GPU Language to use: Either CUDA or OpenCL (default)') 4962247410dSMatthew G. Knepley parser.add_argument('--plots', nargs='+', help='List of plots to show') 497683aebbfSMatthew G Knepley subparsers = parser.add_subparsers(help='DM types') 498eda8839fSMatthew G Knepley 499683aebbfSMatthew G Knepley parser_dmda = subparsers.add_parser('DMDA', help='Use a DMDA for the problem geometry') 500683aebbfSMatthew G Knepley parser_dmda.add_argument('--size', nargs='+', default=['10'], help='Grid size (implementation dependent)') 501683aebbfSMatthew G Knepley parser_dmda.add_argument('--comp', type = int, default='1', help='Number of field components') 502683aebbfSMatthew G Knepley parser_dmda.add_argument('runs', nargs='*', help='Run descriptions: <name>=<args>') 503683aebbfSMatthew G Knepley 5042247410dSMatthew G. Knepley parser_dmmesh = subparsers.add_parser('DMPlex', help='Use a DMPlex for the problem geometry') 505683aebbfSMatthew G Knepley parser_dmmesh.add_argument('--dim', type = int, default='2', help='Spatial dimension') 5062247410dSMatthew G. Knepley parser_dmmesh.add_argument('--refine', type = int, default='0', help='Number of refinements') 507683aebbfSMatthew G Knepley parser_dmmesh.add_argument('runs', nargs='*', help='Run descriptions: <name>=<args>') 508eda8839fSMatthew G Knepley 509eda8839fSMatthew G Knepley args = parser.parse_args() 510683aebbfSMatthew G Knepley if hasattr(args, 'comp'): 511683aebbfSMatthew G Knepley args.dmType = 'DMDA' 512683aebbfSMatthew G Knepley else: 5132247410dSMatthew G. Knepley args.dmType = 'DMPlex' 514683aebbfSMatthew G Knepley 5152247410dSMatthew G. Knepley ex = PETScExample(args.library, args.num, preload='off') 5163973d8daSKarl Rupp if args.gpulang == 'CUDA': 5173973d8daSKarl Rupp source = ex.petsc.source(args.library, args.num, '.cu') 5183973d8daSKarl Rupp else: 5193973d8daSKarl Rupp source = ex.petsc.source(args.library, args.num, 'OpenCL.c') # Using the convention of OpenCL code residing in source files ending in 'OpenCL.c' (at least for snes/ex52) 5203428b40fSMatthew G Knepley sizes = {} 5213428b40fSMatthew G Knepley times = {} 5222247410dSMatthew G. Knepley errors = {} 5232247410dSMatthew G. Knepley meshExp = {} 5242247410dSMatthew G. Knepley timeExp = {} 5253428b40fSMatthew G Knepley events = {} 526d006b754SMatthew G Knepley log = not args.daemon 527d006b754SMatthew G Knepley 528d006b754SMatthew G Knepley if args.daemon: 529d006b754SMatthew G Knepley import daemon 5305b6bfdb9SJed Brown print('Starting daemon') 531d006b754SMatthew G Knepley daemon.createDaemon('.') 532683aebbfSMatthew G Knepley 533eda8839fSMatthew G Knepley for run in args.runs: 534eda8839fSMatthew G Knepley name, stropts = run.split('=', 1) 535eda8839fSMatthew G Knepley opts = dict([t if len(t) == 2 else (t[0], None) for t in [arg.split('=', 1) for arg in stropts.split(' ')]]) 5362247410dSMatthew G. Knepley #opts['log_view'] = 'summary.dat' if args.batch else ':'+args.module+'%s%d.py:ascii_info_detail' 5372247410dSMatthew G. Knepley opts['log_view'] = 'summary.dat' if args.batch else ':'+args.module+'%s%d.csv:ascii_csv' 5382247410dSMatthew G. Knepley meshExp[name] = float(opts['meshExp']) 5392247410dSMatthew G. Knepley timeExp[name] = float(opts['timeExp']) 5403428b40fSMatthew G Knepley sizes[name] = [] 5413428b40fSMatthew G Knepley times[name] = [] 5422247410dSMatthew G. Knepley errors[name] = [] 5433428b40fSMatthew G Knepley events[name] = {} 5444992a29cSMatthew G. Knepley getattr(__main__, 'run_'+args.dmType)(ex, name, opts, args, sizes, times, events, log=log, execute=(not args.plotOnly)) 545d006b754SMatthew G Knepley outputData(sizes, times, events) 5462247410dSMatthew G. Knepley if not args.batch and log: 5472247410dSMatthew G. Knepley for plot in args.plots: 5482247410dSMatthew G. Knepley print('Plotting ',plot) 5492247410dSMatthew G. Knepley getattr(__main__, 'plot'+plot)(args.library, args.num, args.events, sizes, times, errors, events) 5502247410dSMatthew G. Knepley 5512247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --events SNESSolve --plots MeshConvergence WorkPrecision WorkPrecisionPareto --num 5 DMDA --size 32 64 128 256 512 1024 --comp 1 GMRES/ILU0="snes_monitor par=0.0 ksp_rtol=1.0e-9 mms=2 ksp_type=gmres pc_type=ilu meshExp=2.0 timeExp=-0.5" GMRES/LU="snes_monitor par=0.0 ksp_rtol=1.0e-9 mms=2 ksp_type=gmres pc_type=lu meshExp=2.0 timeExp=-0.75" GMRES/GAMG="snes_monitor par=0.0 ksp_rtol=1.0e-9 mms=2 ksp_type=gmres pc_type=gamg meshExp=2.0 timeExp=-1.0" 5522247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --stage "ConvEst Refinement Level %d" --events SNESSolve "ConvEst Error" --plots MeshConvergence WorkPrecision WorkPrecisionPareto --num 13 DMPlex --refine 5 --dim 2 GMRES/ILU0="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=ilu meshExp=2.0 timeExp=-0.75 dm_refine=4 potential_petscspace_order=1" GMRES/LU="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=lu meshExp=2.0 timeExp=-0.75 dm_refine=4 potential_petscspace_order=1" GMRES/GAMG="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=gamg meshExp=2.0 timeExp=-1.0 dm_refine=4 potential_petscspace_order=1" 5532247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --stage "ConvEst Refinement Level %d" --events SNESSolve "ConvEst Error" --plots MeshConvergence WorkPrecision WorkPrecisionPareto --num 13 DMPlex --refine 5 --dim 2 GMRES/ILU0="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=ilu meshExp=3.0 timeExp=-0.75 dm_refine=3 potential_petscspace_order=2" GMRES/LU="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=lu meshExp=3.0 timeExp=-0.75 dm_refine=3 potential_petscspace_order=2" GMRES/GAMG="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=gamg meshExp=3.0 timeExp=-1.0 dm_refine=3 potential_petscspace_order=2" 5542247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --stage "ConvEst Refinement Level %d" --events SNESSolve "ConvEst Error" --plots MeshConvergence WorkPrecision WorkPrecisionPareto --num 13 DMPlex --refine 5 --dim 2 GMRES/ILU0="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=ilu meshExp=3.0 timeExp=-0.75 dm_refine=3 potential_petscspace_order=2" GMRES/LU="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=lu meshExp=3.0 timeExp=-0.75 dm_refine=3 potential_petscspace_order=2" GMRES/GAMG="snes_monitor ksp_rtol=1.0e-9 ksp_type=gmres pc_type=gamg meshExp=3.0 timeExp=-1.0 dm_refine=3 potential_petscspace_order=2" 5552247410dSMatthew G. Knepley 5562247410dSMatthew G. Knepley# Old GPU benchmarks 557683aebbfSMatthew G Knepley# Benchmark for ex50 558683aebbfSMatthew G Knepley# ./src/benchmarks/benchmarkExample.py --events VecMDot VecMAXPY KSPGMRESOrthog MatMult VecCUSPCopyTo VecCUSPCopyFrom MatCUSPCopyTo --num 50 DMDA --size 10 20 50 100 --comp 4 CPU='pc_type=none mat_no_inode dm_vec_type=seq dm_mat_type=seqaij' GPU='pc_type=none mat_no_inode dm_vec_type=seqcusp dm_mat_type=seqaijcusp cusp_synchronize' 559683aebbfSMatthew G Knepley# Benchmark for ex52 5602247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --events IntegBatchCPU IntegBatchGPU IntegGPUOnly --num 52 DMPlex --refine 0.0625 0.00625 0.000625 0.0000625 --blockExp 4 --order=1 CPU='dm_view show_residual=0 compute_function batch' GPU='dm_view show_residual=0 compute_function batch gpu gpu_batches=8' 5612247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --events IntegBatchCPU IntegBatchGPU IntegGPUOnly --num 52 DMPlex --refine 0.0625 0.00625 0.000625 0.0000625 --blockExp 4 --order=1 --operator=elasticity CPU='dm_view op_type=elasticity show_residual=0 compute_function batch' GPU='dm_view op_type=elasticity show_residual=0 compute_function batch gpu gpu_batches=8' 5622247410dSMatthew G. Knepley# ./src/benchmarks/benchmarkExample.py --events IntegBatchCPU IntegBatchGPU IntegGPUOnly --num 52 DMPlex --dim=3 --refine 0.0625 0.00625 0.000625 0.0000625 --blockExp 4 --order=1 CPU='dim=3 dm_view show_residual=0 compute_function batch' GPU='dim=3 dm_view show_residual=0 compute_function batch gpu gpu_batches=8' 563