1#!/usr/bin/env python3 2""" 3Parse the PETSc tutorial or test file (example) and return a dictionary containing information about the tests to be run for the example 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42from gmakegen import * 43sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 44 45import inspect 46thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 47maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 48sys.path.insert(0,maintdir) 49 50# These are special keys describing build 51buildkeys="requires TODO SKIP depends".split() 52 53acceptedkeys=set("test nsize requires command suffix diff_args args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor env temporaries".split()) 54appendlist="args diff_args requires comments env".split() 55 56import re 57 58def getDefaultOutputFileRoot(testname): 59 """ 60 Given testname, give DefaultRoot and DefaultOutputFilename 61 e.g., runex1 gives ex1_1, output/ex1_1.out 62 """ 63 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 64 if not "_" in defroot: defroot=defroot+"_1" 65 return defroot 66 67def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 68 """ 69 Go through and remove a level of indentation 70 Also strip of trailing whitespace 71 """ 72 # The first entry should be test: but it might be indented. 73 ext = getlangext(srcfile) 74 stripstr=" " 75 if len(fileNums)>0: lineNum=fileNums[0]-1 76 for lline in block.split("\n"): 77 if len(fileNums)>0: lineNum+=1 78 line=lline[1:] if lline.startswith("!") else lline 79 if not line.strip(): continue 80 if line.strip().startswith('#'): continue 81 if entireBlock: 82 var=line.split(":")[0].strip() 83 if not var in ['test','testset','build']: 84 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 85 nspace=len(line)-len(line.lstrip(stripstr)) 86 newline=line[nspace:] 87 break 88 89 # Strip off any indentation for the whole string and any trailing 90 # whitespace for convenience 91 newTestStr="\n" 92 if len(fileNums)>0: lineNum=fileNums[0]-1 93 firstPass=True 94 for lline in block.split("\n"): 95 if len(fileNums)>0: lineNum+=1 96 line=lline[1:] if lline.startswith("!") else lline 97 if not line.strip(): continue 98 if line.strip().startswith('#'): 99 newTestStr+=line+'\n' 100 else: 101 newline=line[nspace:] 102 newTestStr+=newline.rstrip()+"\n" 103 # Do some basic indentation checks 104 if entireBlock: 105 # Don't need to check comment lines 106 if line.strip().startswith('#'): continue 107 if not newline.startswith(" "): 108 var=newline.split(":")[0].strip() 109 if not var in ['test','testset','build']: 110 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 111 if len(fileNums)>0: 112 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 113 else: 114 raise Exception(err) 115 else: 116 var=line.split(":")[0].strip() 117 if var in ['test','testset','build']: 118 subnspace=len(line)-len(line.lstrip(stripstr)) 119 if firstPass: 120 firstsubnspace=subnspace 121 firstPass=False 122 else: 123 if firstsubnspace!=subnspace: 124 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 125 if len(fileNums)>0: 126 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 127 else: 128 raise Exception(err) 129 130 # Allow line continuation character '\' 131 return newTestStr.replace('\\\n', ' ') 132 133def parseLoopArgs(varset): 134 """ 135 Given: String containing loop variables 136 Return: tuple containing separate/shared and string of loop vars 137 """ 138 keynm=varset.split("{{")[0].strip().lstrip('-') 139 if not keynm.strip(): keynm='nsize' 140 lvars=varset.split('{{')[1].split('}')[0] 141 suffx=varset.split('{{')[1].split('}')[1] 142 ftype='separate' if suffx.startswith('separate') else 'shared' 143 return keynm,lvars,ftype 144 145def _getLoopVars(testDict): 146 """ 147 Given: dictionary that may have 148 Return: Variables that cause a test split 149 """ 150 vals=None 151 loopVars={} 152 loopVars['separate']=[] 153 loopVars['shared']=[] 154 # Check nsize 155 if 'nsize' in testDict: 156 varset=testDict['nsize'] 157 if '{{' in varset: 158 keynm,lvars,ftype=parseLoopArgs(varset) 159 if ftype=='separate': loopVars['separate'].append(keynm) 160 161 # Now check args 162 if 'args' not in testDict: return loopVars 163 for varset in re.split(r'(^|\W)-(?=[a-zA-Z])',testDict['args']): 164 if not varset.strip(): continue 165 if '{{' in varset: 166 # Assuming only one for loop per var specification 167 keynm,lvars,ftype=parseLoopArgs(varset) 168 loopVars[ftype].append(keynm) 169 170 return loopVars 171 172def _getNewArgs(args,separate=True): 173 """ 174 Given: String that has args that might have loops in them 175 Return: All of the arguments/values that do not have 176 for 'separate output' in for loops 177 unless separate=False 178 """ 179 newargs='' 180 if not args.strip(): return args 181 for varset in re.split(r'(^|\W)-(?=[a-zA-Z])',args): 182 if not varset.strip(): continue 183 if '{{' in varset: 184 if separate: 185 if 'separate' in varset: continue 186 else: 187 if 'separate' not in varset: continue 188 189 newargs+="-"+varset.strip()+" " 190 191 return newargs 192 193def _getVarVals(findvar,testDict): 194 """ 195 Given: variable that is either nsize or in args 196 Return: Values to loop over and the other arguments 197 Note that we keep the other arguments even if they have 198 for loops to enable stepping through all of the for lops 199 """ 200 save_vals=None 201 if findvar=='nsize': 202 varset=testDict[findvar] 203 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 204 else: 205 varlist=[] 206 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 207 if not varset.strip(): continue 208 if '{{' not in varset: continue 209 keyvar,vals,ftype=parseLoopArgs(varset) 210 if keyvar==findvar: 211 save_vals=vals 212 213 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 214 return save_vals 215 216def genTestsSeparateTestvars(intests,indicts,final=False): 217 """ 218 Given: testname, sdict with 'separate_testvars 219 Return: testnames,sdicts: List of generated tests 220 The tricky part here is the {{ ... }separate output} 221 that can be used multiple times 222 """ 223 testnames=[]; sdicts=[] 224 for i in range(len(intests)): 225 testname=intests[i]; sdict=indicts[i]; i+=1 226 loopVars=_getLoopVars(sdict) 227 if len(loopVars['shared'])>0 and not final: 228 # Need to remove shared loop vars and push down to subtests 229 if 'subtests' in sdict: 230 for varset in re.split(r'(^|\W)-(?=[a-zA-Z])',sdict['args']): 231 if '{{' in varset: 232 for stest in sdict['subtests']: 233 if 'args' in sdict[stest]: 234 sdict[stest]['args']+=' -'+varset 235 else: 236 sdict[stest]['args']="-"+varset 237 sdict['args']=_getNewArgs(sdict['args'],separate=False) 238 if len(loopVars['separate'])>0: 239 sep_dicts=[sdict.copy()] 240 if 'args' in sep_dicts[0]: 241 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 242 sep_testnames=[testname] 243 for kvar in loopVars['separate']: 244 kvals=_getVarVals(kvar,sdict) 245 246 # Have to do loop over previous var/val combos as well 247 # and accumulate as we go 248 val_testnames=[]; val_dicts=[] 249 for val in kvals.split(): 250 gensuffix="_"+kvar+"-"+val.replace(',','__') 251 for kvaltestnm in sep_testnames: 252 val_testnames.append(kvaltestnm+gensuffix) 253 for kv in sep_dicts: 254 kvardict=kv.copy() 255 # If the last var then we have the final version 256 if 'suffix' in sdict: 257 kvardict['suffix']+=gensuffix 258 else: 259 kvardict['suffix']=gensuffix 260 if kvar=='nsize': 261 kvardict[kvar]=val 262 else: 263 kvardict['args']+="-"+kvar+" "+val+" " 264 val_dicts.append(kvardict) 265 sep_testnames=val_testnames 266 sep_dicts=val_dicts 267 testnames+=sep_testnames 268 sdicts+=sep_dicts 269 else: 270 # These are plain vanilla tests (no subtests, no loops) that 271 # do not have a suffix. This makes the targets match up with 272 # the output file (testname_1.out) 273 if final: 274 if '_' not in testname: testname+='_1' 275 testnames.append(testname) 276 sdicts.append(sdict) 277 return testnames,sdicts 278 279def genTestsSubtestSuffix(testnames,sdicts): 280 """ 281 Given: testname, sdict with separate_testvars 282 Return: testnames,sdicts: List of generated tests 283 """ 284 tnms=[]; sdcts=[] 285 for i in range(len(testnames)): 286 testname=testnames[i] 287 rmsubtests=[]; keepSubtests=False 288 if 'subtests' in sdicts[i]: 289 for stest in sdicts[i]["subtests"]: 290 if 'suffix' in sdicts[i][stest]: 291 rmsubtests.append(stest) 292 gensuffix="_"+sdicts[i][stest]['suffix'] 293 newtestnm=testname+gensuffix 294 tnms.append(newtestnm) 295 newsdict=sdicts[i].copy() 296 del newsdict['subtests'] 297 # Have to hand update 298 # Append 299 for kup in appendlist: 300 if kup in sdicts[i][stest]: 301 if kup in sdicts[i]: 302 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 303 else: 304 newsdict[kup]=sdicts[i][stest][kup] 305 # Promote 306 for kup in acceptedkeys: 307 if kup in appendlist: continue 308 if kup in sdicts[i][stest]: 309 newsdict[kup]=sdicts[i][stest][kup] 310 # Cleanup 311 for st in sdicts[i]["subtests"]: del newsdict[st] 312 sdcts.append(newsdict) 313 else: 314 keepSubtests=True 315 else: 316 tnms.append(testnames[i]) 317 sdcts.append(sdicts[i]) 318 # If a subtest without a suffix exists, then save it 319 if keepSubtests: 320 tnms.append(testnames[i]) 321 newsdict=sdicts[i].copy() 322 # Prune the tests to prepare for keeping 323 for rmtest in rmsubtests: 324 newsdict['subtests'].remove(rmtest) 325 del newsdict[rmtest] 326 sdcts.append(newsdict) 327 i+=1 328 return tnms,sdcts 329 330def splitTests(testname,sdict): 331 """ 332 Given: testname and dictionary generated from the YAML-like definition 333 Return: list of names and dictionaries corresponding to each test 334 given that the YAML-like language allows for multiple tests 335 """ 336 337 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 338 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 339 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 340 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 341 342 # Because I am altering the list, I do this in passes. Inelegant 343 344 return testnames, sdicts 345 346 347def testSplit(striptest): 348 """ 349 Split up a test into lines, but use a shell parser to detect when newlines are within quotation marks 350 and keep those together 351 """ 352 import shlex 353 354 sl = shlex.shlex() 355 sl.whitespace_split = True # only split at whitespace 356 sl.commenters = '' 357 sl.push_source(striptest) 358 last_pos = sl.instream.tell() 359 try: 360 last_token = sl.read_token() 361 except ValueError: 362 print(striptest) 363 raise ValueError 364 last_line = '' 365 while last_token != '': 366 new_pos = sl.instream.tell() 367 block = striptest[last_pos:new_pos] 368 token_start = block.find(last_token) 369 leading = block[0:token_start] 370 trailing = block[(token_start + len(last_token)):] 371 leading_split = leading.split('\n') 372 if len(leading_split) > 1: 373 yield last_line 374 last_line = '' 375 last_line += leading_split[-1] 376 last_line += last_token 377 trailing_split = trailing.split('\n') 378 last_line += trailing_split[0] 379 if len(trailing_split) > 1: 380 yield last_line 381 last_line = '' 382 last_pos = new_pos 383 try: 384 last_token = sl.read_token() 385 except ValueError: 386 print(striptest) 387 raise ValueError 388 yield last_line 389 390 391def parseTest(testStr,srcfile,verbosity): 392 """ 393 This parses an individual test 394 Our YAML-like language is hierarchial so should use a state machine in the general case, 395 but in practice we only support two levels of test: 396 """ 397 basename=os.path.basename(srcfile) 398 # Handle the new at the beginning 399 bn=re.sub("new_","",basename) 400 # This is the default 401 testname="run"+getlangsplit(bn) 402 403 # Tests that have default everything (so empty effectively) 404 if len(testStr)==0: 405 if '_' not in testname: testname+='_1' 406 return [testname], [{}] 407 408 striptest=_stripIndent(testStr,srcfile) 409 410 # go through and parse 411 subtestnum=0 412 subdict={} 413 comments=[] 414 indentlevel=0 415 for ln in testSplit(striptest): 416 line=ln.split('#')[0].rstrip() 417 if verbosity>2: print(line) 418 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 419 if comment: comments.append(comment) 420 if not line.strip(): continue 421 lsplit=line.split(':') 422 if len(lsplit)==0: 423 raise Exception("\n\nError in test harness parsing file: "+srcfile+"\nMissing : in line: "+line) 424 indentcount=lsplit[0].count(" ") 425 var=lsplit[0].strip() 426 val=line[line.find(':')+1:].strip() 427 if not var in acceptedkeys: 428 raise Exception("\n\nError in test harness parsing file: "+srcfile+"\n"+var+" from: "+line+" is not a valid keyword") 429 # Start by seeing if we are in a subtest 430 if line.startswith(" "): 431 if not 'subtestname' in locals(): 432 raise Exception("\n\nError in test harness parsing file: "+srcfile+"\nInvalid indentation at line:"+line) 433 if var in subdict[subtestname]: 434 subdict[subtestname][var]+=" "+val 435 else: 436 subdict[subtestname][var]=val 437 if not indentlevel: indentlevel=indentcount 438 #if indentlevel!=indentcount: print("Error in indentation:", ln) 439 # Determine subtest name and make dict 440 elif var=="test": 441 subtestname="test"+str(subtestnum) 442 subdict[subtestname]={} 443 if "subtests" not in subdict: subdict["subtests"]=[] 444 subdict["subtests"].append(subtestname) 445 subtestnum=subtestnum+1 446 # The rest are easy 447 else: 448 # For convenience, it is sometimes convenient to list twice 449 if var in subdict: 450 if var in appendlist: 451 subdict[var]+=" "+val 452 else: 453 raise Exception("\n\nError in test harness parsing file: "+srcfile+"\n"+var+" entered twice: "+line) 454 else: 455 subdict[var]=val 456 if var=="suffix": 457 if len(val)>0: 458 testname+="_"+val 459 if var == "env" and len(val) == 0: 460 raise Exception("\n\nError in test harness parsing file: "+srcfile+"\nvalue for {}: directive cannot be empty!".format(var)) 461 462 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 463 464 # add in error check: if 'requires' is added to subtests, make sure 'suffix' is also added. 465 if 'subtests' in subdict: 466 for stest in subdict['subtests']: 467 if 'requires' in subdict[stest] and not 'suffix' in subdict[stest]: 468 raise Exception("\n\nError in test harness parsing file: "+srcfile+"\nA test in a testset with 'requires' directive should also have a 'suffix' directive!") 469 470 # A test block can create multiple tests. This does that logic 471 testnames,subdicts=splitTests(testname,subdict) 472 return testnames,subdicts 473 474def parseTests(testStr,srcfile,fileNums,verbosity): 475 """ 476 Parse the YAML-like string describing tests and return 477 a dictionary with the info in the form of: 478 testDict[test][subtest] 479 """ 480 481 testDict={} 482 483 # The first entry should be test: but it might be indented. 484 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 485 if verbosity>2: print(srcfile) 486 487 ## Check and see if we have build requirements 488 addToRunRequirements=None 489 if "\nbuild:" in newTestStr: 490 testDict['build']={} 491 # The file info is already here and need to append 492 Part1=newTestStr.split("build:")[1] 493 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 494 for bkey in buildkeys: 495 if bkey+":" in fileInfo: 496 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 497 #if verbosity>1: bkey+": "+testDict['build'][bkey] 498 # If a runtime requires are put into build, push them down to all run tests 499 # At this point, we are working with strings and not lists 500 if 'requires' in testDict['build']: 501 if 'todo' in testDict['build']['requires'] or 'TODO' in testDict['build']['requires']: 502 raise Exception("Error: Do not list 'TODO' in requires: list it as a field, for example, TODO: XXX is currently unstable\n"+srcfile) 503 addToRunRequirements=testDict['build']['requires'] 504 # People put datafilespath into build, but it needs to be a runtime 505 if 'datafilespath' in testDict['build']['requires']: 506 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 507 testDict['build']['requires']=newreqs.strip() 508 509 510 # Now go through each test. First elem in split is blank 511 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 512 testnames,subdicts=parseTest(test,srcfile,verbosity) 513 for i in range(len(testnames)): 514 if testnames[i] in testDict: 515 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 516 # Add in build requirements that need to be moved 517 if addToRunRequirements: 518 if 'requires' in subdicts[i]: 519 subdicts[i]['requires']+=' '+addToRunRequirements 520 else: 521 subdicts[i]['requires']=addToRunRequirements 522 testDict[testnames[i]]=subdicts[i] 523 524 return testDict 525 526def parseTestFile(srcfile,verbosity): 527 """ 528 Parse single example files and return dictionary of the form: 529 testDict[srcfile][test][subtest] 530 """ 531 debug=False 532 basename=os.path.basename(srcfile) 533 if basename=='makefile': return {} 534 535 curdir=os.path.realpath(os.path.curdir) 536 basedir=os.path.dirname(os.path.realpath(srcfile)) 537 os.chdir(basedir) 538 539 testDict={} 540 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 541 # Handle Windows issues 542 if not os.linesep == "\n": fileStr=fileStr.replace(os.linesep,"\n") 543 544 ## Start with doing the tests 545 # 546 fsplit=fileStr.split("/*TEST\n")[1:] 547 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 548 # Allow for multiple "/*TEST" blocks even though it really should be 549 # one 550 srcTests=[] 551 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 552 testString=" ".join(srcTests) 553 flen=len(testString.split("\n")) 554 fend=fstart+flen-1 555 fileNums=range(fstart,fend) 556 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 557 # Massage dictionary for build requirements 558 if 'build' in testDict[basename]: 559 testDict[basename].update(testDict[basename]['build']) 560 del testDict[basename]['build'] 561 562 563 os.chdir(curdir) 564 return testDict 565 566def parseTestDir(directory,verbosity): 567 """ 568 Parse single example files and return dictionary of the form: 569 testDict[srcfile][test][subtest] 570 """ 571 curdir=os.path.realpath(os.path.curdir) 572 basedir=os.path.realpath(directory) 573 os.chdir(basedir) 574 575 tDict={} 576 for test_file in sorted(glob.glob("new_ex*.*")): 577 tDict.update(parseTestFile(test_file,verbosity)) 578 579 os.chdir(curdir) 580 return tDict 581 582def printExParseDict(rDict): 583 """ 584 This is useful for debugging 585 """ 586 indent=" " 587 for sfile in rDict: 588 print(sfile) 589 sortkeys=list(rDict[sfile].keys()) 590 sortkeys.sort() 591 for runex in sortkeys: 592 if runex == 'requires': 593 print(indent+runex+':'+str(rDict[sfile][runex])) 594 continue 595 print(indent+runex) 596 if type(rDict[sfile][runex])==bytes: 597 print(indent*2+rDict[sfile][runex]) 598 else: 599 for var in rDict[sfile][runex]: 600 if var.startswith("test"): continue 601 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 602 if 'subtests' in rDict[sfile][runex]: 603 for var in rDict[sfile][runex]['subtests']: 604 print(indent*2+var) 605 for var2 in rDict[sfile][runex][var]: 606 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 607 print("\n") 608 return 609 610def main(directory='',test_file='',verbosity=0): 611 612 if directory: 613 tDict=parseTestDir(directory,verbosity) 614 else: 615 tDict=parseTestFile(test_file,verbosity) 616 if verbosity>0: printExParseDict(tDict) 617 618 return 619 620if __name__ == '__main__': 621 import optparse 622 parser = optparse.OptionParser() 623 parser.add_option('-d', '--directory', dest='directory', 624 default="", help='Directory containing files to parse') 625 parser.add_option('-t', '--test_file', dest='test_file', 626 default="", help='Test file, e.g., ex1.c, to parse') 627 parser.add_option('-v', '--verbosity', dest='verbosity', 628 help='Verbosity of output by level: 1, 2, or 3', default=0) 629 opts, extra_args = parser.parse_args() 630 631 if extra_args: 632 import sys 633 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 634 exit(1) 635 if not opts.test_file and not opts.directory: 636 print("test file or directory is required") 637 parser.print_usage() 638 sys.exit() 639 640 # Need verbosity to be an integer 641 try: 642 verbosity=int(opts.verbosity) 643 except: 644 raise Exception("Error: Verbosity must be integer") 645 646 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 647