1#!/usr/bin/python 2 3import fnmatch 4import glob 5import optparse 6import os 7import re 8import sys 9import time 10import types 11 12import inspect 13currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 14sys.path.insert(0,currentdir) 15#import runhtml 16 17class logParse(object): 18 def __init__(self,petsc_dir,logdir,verbosity): 19 20 self.petsc_dir=petsc_dir 21 self.verbosity=verbosity 22 self.logdir=logdir 23 return 24 25 def findSrcfile(self,testname): 26 """ 27 Given a testname of the form runex10_9, try to figure out the source file 28 """ 29 testnm=testname.replace("diff-","") 30 dirpart=testnm.split("-")[0] 31 namepart=testnm.split("-")[1].split("_")[0] 32 # First figure out full directory to source file 33 splt=(re.split("_tests",dirpart) if "_tests" in dirpart 34 else re.split("_tutorials",dirpart)) 35 tdir="tests" if "_tests" in dirpart else "tutorials" 36 filedir=os.path.join(self.petsc_dir,"src", 37 splt[0].replace("_","/"),"examples",tdir) 38 39 # Directory names with "-" cause problems, so more work required 40 if testnm.count('-') > 2: 41 namepart=testnm.split("-")[-1].split("_")[0] 42 splitl=(re.split("_tests",testnm) if "_tests" in testnm 43 else re.split("_tutorials",testnm)) 44 subdir='-'.join(splitl[1].lstrip('_').split('-')[:-1]) 45 filedir=os.path.join(filedir,subdir) 46 47 # Directory names with underscores cause problems, so more work required 48 subdir="" 49 if len(splt)>1: 50 for psub in splt[1].split("_"): 51 subdir+=psub 52 if os.path.isdir(os.path.join(filedir,subdir)): 53 filedir=os.path.join(filedir,subdir) 54 subdir="" 55 continue 56 subdir+="_" 57 58 # See what files exists with guessing the extension 59 base=namepart 60 if base.endswith("f") or base.endswith("f90"): 61 for ext in ['F','F90']: 62 guess=os.path.join(filedir,base+"."+ext) 63 if os.path.exists(guess): return guess 64 else: 65 for ext in ['c','cxx']: 66 guess=os.path.join(filedir,base+"."+ext) 67 if os.path.exists(guess): return guess 68 # Sometimes the underscore is in the executable (ts-tutorials) 69 base=namepart[3:] # I can't remember how this works 70 if base.endswith("f") or base.endswith("f90"): 71 for ext in ['F','F90']: 72 guess=os.path.join(filedir,base+"."+ext) 73 if os.path.exists(guess): return guess 74 else: 75 for ext in ['c','cxx']: 76 guess=os.path.join(filedir,base+"."+ext) 77 if os.path.exists(guess): return guess 78 print(filedir, namepart) 79 print("Warning: Cannot find file for "+testname) 80 return None 81 82 def getGitPerson(self,fullFileName): 83 """ 84 Given a testname, find the file in petsc_dir and find out who did the last 85 commit 86 """ 87 git_authorname_cmd='git log -1 --pretty=format:"%an <%ae>" '+fullFileName 88 try: 89 #git_blame_cmd = 'git blame -w -M --line-porcelain --show-email -L '+' -L '.join(pairs)+' '+key[0]+' -- '+key[1] 90 fh=os.popen(git_authorname_cmd) 91 output=fh.read(); fh.close 92 except: 93 raise Exception("Error running: "+git_authorname_cmd) 94 return output 95 96 def getTestDict(self,logDict): 97 """ 98 Summarize all of the logfile data by test and then errors 99 Want to know if same error occurs on multiple machines 100 """ 101 testDict={} 102 testDict['info']={'branch':logDict['branch']} 103 testDict['info']['errors']={} 104 for logfile in logDict: 105 if logfile=='branch': continue 106 lfile=logfile.replace("examples_","").replace(".log","") 107 for test in logDict[logfile]: 108 filename=self.findSrcfile(test) 109 if not filename: continue 110 fname=os.path.relpath(filename,self.petsc_dir).replace("src/","") 111 testname=test.replace("diff-","") if test.startswith("diff-") else test 112 error=logDict[logfile][test].strip() 113 error=error if test==testname else "Diff errors:\n"+error 114 if error=="": error="No error" 115 # Organize by filename 116 if not fname in testDict: 117 testDict[fname]={} 118 testDict[fname]['gitPerson']=str(self.getGitPerson(filename)) 119 # Now organize by test and errors 120 if testname in testDict[fname]: 121 if error in testDict[fname][testname]['errors']: 122 testDict[fname][testname]['errors'][error].append(lfile) 123 else: 124 testDict[fname][testname]['errors'][error]=[lfile] 125 else: 126 #print testname+","+fname+"," 127 testDict[fname][testname]={} 128 # We'll be adding other keys later 129 testDict[fname][testname]['errors']={} 130 testDict[fname][testname]['errors'][error]=[lfile] 131 # Create additional datastructure to sort by errors 132 if error in testDict['info']['errors']: 133 testDict['info']['errors'][error][testname]=fname 134 else: 135 testDict['info']['errors'][error]={testname:fname} 136 137 # Place holder for later -- keep log of analysis 138 for fname in testDict: 139 if fname=='info': continue 140 for test in testDict[fname]: 141 if test=='gitPerson': continue 142 testDict[fname][test]['ndays']=0 143 testDict[fname][test]['fdate']='Date' 144 145 return testDict 146 147 def writeSummaryLog(self,testDict,outprefix): 148 """ 149 Just do a simple pretty print 150 """ 151 branch=testDict['info']['branch'] 152 fh=open(outprefix+branch+".csv","w") 153 c=',' 154 for fname in testDict: 155 if fname=='info': continue 156 for test in testDict[fname]: 157 if test=='gitPerson': continue 158 ndays=testDict[fname][test]['ndays'] 159 fdate=testDict[fname][test]['fdate'] 160 fh.write(fname+c+test+c+str(ndays)+fdate) 161 162 fh.close() 163 return 164 165 166 def printTestDict(self,testDict): 167 """ 168 Just do a simple pretty print 169 """ 170 indent=" " 171 for fname in testDict: 172 if fname=='info': continue 173 for test in testDict[fname]: 174 print("\n ----------------------------------------------------") 175 print(test) 176 print(indent+testDict[fname][test]['gitPerson']) 177 for error in testDict[fname][test]['errors']: 178 print("\n ----- ") 179 print(2*indent+" ".join(testDict[fname][test]['errors'][error])) 180 print(2*indent+error) 181 182 return testDict 183 184 def getLogLink(self,arch): 185 """ 186 For the html for showing the log link 187 """ 188 return '<td class="border"><a href=\"examples_'+arch+'.log\">[log]</a></td>' 189 190 def writeHTML(self,testDict,outprefix): 191 """ 192 Put it into an HTML table 193 """ 194 import htmltemplate 195 196 branch=testDict['info']['branch'] 197 branchtitle="PETSc Examples ("+branch+")" 198 branchhtml=branch+".html" 199 htmlfiles=[] 200 for hf in "sortByPkg sortByPerson sortByErrors".split(): 201 htmlfiles.append(outprefix+branch+'-'+hf+".html") 202 203 # ---------------------------------------------------------------- 204 # This is by package and test name 205 ofh=open(htmlfiles[0],"w") 206 ofh.write(htmltemplate.getHeader(branchtitle+" - Sort By Package/Test")) 207 208 ofh.write('See also: \n') 209 ofh.write('<a href="'+branchhtml+'">'+branchhtml+'</a> \n') 210 ofh.write('  \n') 211 ofh.write('<a href="'+htmlfiles[1]+'">'+htmlfiles[1]+'</a>\n\n') 212 ofh.write('  \n') 213 ofh.write('<a href="'+htmlfiles[2]+'">'+htmlfiles[2]+'</a><br><br>\n\n') 214 215 pkgs="sys vec mat dm ksp snes ts tao".split() 216 ofh.write('Packages: \n') 217 for pkg in pkgs: 218 ofh.write('<b><a href="#'+pkg+'">'+pkg+'</a></b>\n') 219 ofh.write("</span></center><br>\n") 220 221 ofh.write("<center><table>\n") 222 223 # sort by example 224 allGitPersons={} 225 for pkg in pkgs: 226 ofh.write('<tr><th class="gray" colspan=4></th></tr>\n') 227 ofh.write('<tr id="'+pkg+'"><th colspan=4>'+pkg+' Package</th></tr>\n') 228 ofh.write("\n\n") 229 ofh.write("<tr><th>Test Name</th><th>Errors</th><th>Arch</th><th>Log</th></tr>\n") 230 ofh.write("\n\n") 231 for fname in testDict: 232 if fname=='info': continue 233 if not fname.startswith(pkg): continue # Perhaps could be more efficient 234 gp=testDict[fname]['gitPerson'] 235 gitPerson=gp.replace("<","<").replace("<",">") 236 gpName=gp.split("<")[0].strip(); gpLastName=gpName.split()[-1] 237 if not gpLastName in allGitPersons: 238 allGitPersons[gpLastName]=(gp,gpName) 239 permlink=htmlfiles[0]+'#'+fname 240 plhtml='<a id="'+permlink+'" href="'+permlink+'"> (permlink)</a>' 241 ofh.write('<tr><th colspan="4">'+fname+"    ("+gitPerson+') '+plhtml+'</th></tr>\n\n') 242 for test in testDict[fname]: 243 if test=='gitPerson': continue 244 i=0 245 for error in testDict[fname][test]['errors']: 246 ofh.write('<!-- New row -->\n') 247 248 teststr='<td class="border">'+test+'</td>' if i==0 else '<td></td>' 249 i+=1 250 arches=testDict[fname][test]['errors'][error] 251 rsnum=len(arches) 252 # Smaller arch string looks nice 253 archstr=[arch.replace(branch+'_','').replace("arch-",'') for arch in arches] 254 255 comstr="<pre>"+error+"</pre>" if error else "No error" 256 comstr='<td class="border" rowspan="'+str(rsnum)+'">'+comstr+'</td>' 257 258 logstr=self.getLogLink(arches[0]) 259 ofh.write('<tr>'+teststr+comstr+'<td class="border">'+archstr[0]+'</td>'+logstr+'</tr>\n') 260 # Log files are then hanging 261 if len(arches)>1: 262 for j in range(1,rsnum): 263 logstr=self.getLogLink(arches[j]) 264 ofh.write("<tr><td></td> <td>"+archstr[j]+"</td> "+logstr+"</tr>\n") 265 ofh.write("\n\n") 266 267 ofh.write("</table>\n<br>\n") 268 ofh.close() 269 270 # ---------------------------------------------------------------- 271 # This is by person 272 ofh=open(htmlfiles[1],"w") 273 ofh.write(htmltemplate.getHeader(branchtitle+" - Sort By Person")) 274 ofh.write("\n\n") 275 276 ofh.write('See also: \n') 277 ofh.write('<a href="'+branchhtml+'">'+branchhtml+'</a> \n') 278 ofh.write('  \n') 279 ofh.write('<a href="'+htmlfiles[0]+'">'+htmlfiles[0]+'</a> \n') 280 ofh.write('  \n') 281 ofh.write('<a href="'+htmlfiles[2]+'">'+htmlfiles[2]+'</a><br><br>\n') 282 283 happyShinyPeople=allGitPersons.keys() 284 happyShinyPeople.sort() # List alphabetically by last name 285 286 ofh.write('People: \n') 287 for person in happyShinyPeople: 288 (gpFullName,gpName)=allGitPersons[person] 289 ofh.write('<a href="#'+person+'">'+gpName+' </a>  \n') 290 ofh.write("</span></center><br>\n") 291 292 293 # sort by person 294 ofh.write("<center><table>\n") 295 for person in happyShinyPeople: 296 (gpFullName,gpName)=allGitPersons[person] 297 ofh.write('<tr><th class="gray" colspan=4></th></tr>\n') 298 ofh.write('<tr id="'+person+'"><th colspan=4>'+gpFullName+'</th></tr>\n') 299 ofh.write("\n\n") 300 ofh.write("<tr><th>Test Name</th><th>Error</th><th></th><th>Arch</th></tr>\n") 301 ofh.write("\n\n") 302 for fname in testDict: 303 if fname=='info': continue 304 gitPerson=testDict[fname]['gitPerson'].replace("<","<").replace("<",">") 305 if not gitPerson.startswith(gpName): continue 306 permlink=htmlfiles[0]+'#'+fname 307 plhtml=' <a id="'+permlink+'" href="'+permlink+'"> (permlink)</a>' 308 ofh.write('<tr><th colspan="4">'+fname+plhtml+'</th></tr>\n\n') 309 for test in testDict[fname]: 310 if test=='gitPerson': continue 311 i=0 312 for error in testDict[fname][test]['errors']: 313 ofh.write('<!-- New row -->\n') 314 315 teststr='<td class="border">'+test+'</td>' if i==0 else '<td></td>' 316 i+=1 317 arches=testDict[fname][test]['errors'][error] 318 rsnum=len(arches) 319 archstr=[arch.replace(branch+'_','').replace("arch-",'') for arch in arches] 320 comstr="<pre>"+error+"</pre>" if error else "No error" 321 comstr='<td class="border" rowspan="'+str(rsnum)+'">'+comstr+'</td>' 322 logstr=self.getLogLink(arches[0]) 323 324 ofh.write('<tr>'+teststr+comstr+'<td class="border">'+archstr[0]+'</td>'+logstr+'</tr>\n') 325 if len(arches)>1: 326 for j in range(1,rsnum): 327 logstr=self.getLogLink(arches[j]) 328 ofh.write("<tr> <td></td> <td>"+archstr[j]+"</td> "+logstr+"</tr>\n") 329 ofh.write("\n\n") 330 331 ofh.write("</table>") 332 ofh.close() 333 334 # ---------------------------------------------------------------- 335 # This is by errors 336 ofh=open(htmlfiles[2],"w") 337 ofh.write(htmltemplate.getHeader(branchtitle+" - Sort By Errors")) 338 ofh.write("\n\n") 339 340 ofh.write('See also: \n') 341 ofh.write('<a href="'+branchhtml+'">'+branchhtml+'</a> \n') 342 ofh.write('  \n') 343 ofh.write('<a href="'+htmlfiles[0]+'">'+htmlfiles[0]+'</a> \n') 344 ofh.write('  \n') 345 ofh.write('<a href="'+htmlfiles[1]+'">'+htmlfiles[1]+'</a><br><br>\n') 346 ofh.write("</span></center><br>\n") 347 348 349 # sort by error 350 ofh.write("<center><table>\n") 351 ofh.write("<tr><th>Error</th><th>Test Name</th><th></th><th>Arch</th></tr>\n") 352 for error in testDict['info']['errors']: 353 ofh.write('<tr><th class="gray" colspan=4></th></tr>\n') 354 ofh.write("\n\n") 355 i=0 356 comstr="<pre>"+error+"</pre>" if error else "No error" 357 comstr='<td class="border">'+comstr+'</td>' 358 for test in testDict['info']['errors'][error]: 359 fname=testDict['info']['errors'][error][test] 360 361 permlink=htmlfiles[0]+'#'+fname 362 plhtml=' <a id="'+permlink+'" href="'+permlink+'"> (permlink)</a>' 363 364 arches=testDict[fname][test]['errors'][error] 365 rsnum=len(arches) 366 if i>0: comstr='<td></td>' 367 i+=1 368 teststr='<td class="border" rowspan="'+str(rsnum)+'">'+test+'</td>' 369 archstr=[arch.replace(branch+'_','').replace("arch-",'') for arch in arches] 370 logstr=self.getLogLink(arches[0]) 371 ofh.write('<tr>'+comstr+teststr+'<td class="border">'+archstr[0]+'</td>'+logstr+'</tr>\n') 372 if len(arches)>1: 373 for j in range(1,rsnum): 374 logstr=self.getLogLink(arches[j]) 375 ofh.write("<tr> <td></td> <td>"+archstr[j]+"</td> "+logstr+"</tr>\n") 376 ofh.write("\n\n") 377 378 ofh.write("</table>") 379 ofh.close() 380 381 382 383 return 384 385 def doLogFiles(self,branch): 386 """ 387 Go through all of the log files and call the parser for each one 388 Get a simple dictionary for each logfile -- later we process 389 to make nice printouts 390 """ 391 logDict={'branch':branch} 392 startdir=os.path.abspath(os.path.curdir) 393 os.chdir(self.logdir) 394 for logfile in glob.glob('examples_'+branch+'_*.log'): 395 if logfile.startswith('examples_full'): continue 396 logDict[logfile]=self.parseLogFile(logfile) 397 os.chdir(startdir) 398 return logDict 399 400 401 def parseLogFile(self,logfile,printDict=False): 402 """ 403 Do the actual parsing of the file and return 404 a dictionary with all of the failed tests along with why they failed 405 """ 406 lDict={} 407 with open(logfile,"r") as infile: 408 while 1: 409 line=infile.readline() 410 if not line: break 411 if line.startswith("not ok "): 412 last_pos=infile.tell() 413 test=line.replace("not ok ","").strip() 414 errors='' 415 while 1: 416 newline=infile.readline() 417 if newline.startswith('#'): 418 errors=newline.lstrip('#').strip() 419 last_pos=infile.tell() 420 else: 421 break 422 infile.seek(last_pos) # Go back b/c we grabbed another test 423 lDict[test]=errors 424 if printDict: 425 for lkey in lDict: 426 print(lkey) 427 print(" "+lDict[lkey].replace("\n","\n ")) 428 return lDict 429 430 431 432def main(): 433 parser = optparse.OptionParser(usage="%prog [options]") 434 parser.add_option('-f', '--logfile', dest='logfile', 435 help='Parse a single file and print out dictionary for debugging') 436 parser.add_option('-l', '--logdir', dest='logdir', 437 help='Directory where to find the log files') 438 parser.add_option('-p', '--petsc_dir', dest='petsc_dir', 439 help='where to find the git repo', 440 default='') 441 parser.add_option('-o', '--outfile', dest='outfile', 442 help='The output file prefix where the HTML code will be written to', 443 default='examples_summary-') 444 parser.add_option('-v', '--verbosity', dest='verbosity', 445 help='Verbosity of output by level: 1, 2, or 3', 446 default='0') 447 parser.add_option('-b', '--branch', dest='branch', 448 help='Comma delimitted list of branches to parse files of form: examples_<branch>_<arch>.log', 449 default='master,next') 450 options, args = parser.parse_args() 451 452 # Process arguments 453 if len(args) > 0: 454 parser.print_usage() 455 return 456 457 # Need verbosity to be an integer 458 try: 459 verbosity=int(options.verbosity) 460 except: 461 raise Exception("Error: Verbosity must be integer") 462 463 petsc_dir=None 464 if options.petsc_dir: petsc_dir=options.petsc_dir 465 if petsc_dir is None: petsc_dir=os.path.dirname(os.path.dirname(currentdir)) 466 if petsc_dir is None: 467 petsc_dir = os.environ.get('PETSC_DIR') 468 if petsc_dir is None: 469 petsc_dir=os.path.dirname(os.path.dirname(currentdir)) 470 471 if not options.logdir: 472 print("Use -l to specify makefile") 473 return 474 475 logP=logParse(petsc_dir,options.logdir,verbosity) 476 if options.logfile: 477 l=logP.parseLogFile(options.logfile,printDict=True) 478 return 479 else: 480 for b in options.branch.split(','): 481 logDict=logP.doLogFiles(b) 482 testDict=logP.getTestDict(logDict) 483 if verbosity>2: 484 logP.printTestDict(testDict) 485 logP.writeHTML(testDict,options.outfile) 486 logP.writeSummaryLog(testDict,options.outfile) 487 488 return 489 490if __name__ == "__main__": 491 main() 492