1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix diff_args args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args diff_args requires comments".split() 54 55import re 56 57def getDefaultOutputFileRoot(testname): 58 """ 59 Given testname, give DefaultRoot and DefaultOutputFilename 60 e.g., runex1 gives ex1_1, output/ex1_1.out 61 """ 62 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 63 if not "_" in defroot: defroot=defroot+"_1" 64 return defroot 65 66def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 67 """ 68 Go through and remove a level of indentation 69 Also strip of trailing whitespace 70 """ 71 # The first entry should be test: but it might be indented. 72 ext=os.path.splitext(srcfile)[1] 73 stripstr=" " 74 if len(fileNums)>0: lineNum=fileNums[0]-1 75 for lline in block.split("\n"): 76 if len(fileNums)>0: lineNum+=1 77 line=lline[1:] if lline.startswith("!") else lline 78 if not line.strip(): continue 79 if line.strip().startswith('#'): continue 80 if entireBlock: 81 var=line.split(":")[0].strip() 82 if not var in ['test','testset','build']: 83 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 84 nspace=len(line)-len(line.lstrip(stripstr)) 85 newline=line[nspace:] 86 break 87 88 # Strip off any indentation for the whole string and any trailing 89 # whitespace for convenience 90 newTestStr="\n" 91 if len(fileNums)>0: lineNum=fileNums[0]-1 92 firstPass=True 93 for lline in block.split("\n"): 94 if len(fileNums)>0: lineNum+=1 95 line=lline[1:] if lline.startswith("!") else lline 96 if not line.strip(): continue 97 if line.strip().startswith('#'): 98 newTestStr+=line+'\n' 99 else: 100 newline=line[nspace:] 101 newTestStr+=newline.rstrip()+"\n" 102 # Do some basic indentation checks 103 if entireBlock: 104 # Don't need to check comment lines 105 if line.strip().startswith('#'): continue 106 if not newline.startswith(" "): 107 var=newline.split(":")[0].strip() 108 if not var in ['test','testset','build']: 109 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 110 if len(fileNums)>0: 111 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 112 else: 113 raise Exception(err) 114 else: 115 var=line.split(":")[0].strip() 116 if var in ['test','testset','build']: 117 subnspace=len(line)-len(line.lstrip(stripstr)) 118 if firstPass: 119 firstsubnspace=subnspace 120 firstPass=False 121 else: 122 if firstsubnspace!=subnspace: 123 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 124 if len(fileNums)>0: 125 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 126 else: 127 raise Exception(err) 128 129 # Allow line continuation character '\' 130 return newTestStr.replace('\\\n', ' ') 131 132def parseLoopArgs(varset): 133 """ 134 Given: String containing loop variables 135 Return: tuple containing separate/shared and string of loop vars 136 """ 137 keynm=varset.split("{{")[0].strip().lstrip('-') 138 if not keynm.strip(): keynm='nsize' 139 lvars=varset.split('{{')[1].split('}')[0] 140 suffx=varset.split('{{')[1].split('}')[1] 141 ftype='separate' if suffx.startswith('separate') else 'shared' 142 return keynm,lvars,ftype 143 144def _getLoopVars(testDict): 145 """ 146 Given: dictionary that may have 147 Return: Variables that cause a test split 148 """ 149 vals=None 150 loopVars={} 151 loopVars['separate']=[] 152 loopVars['shared']=[] 153 # Check nsize 154 if 'nsize' in testDict: 155 varset=testDict['nsize'] 156 if '{{' in varset: 157 keynm,lvars,ftype=parseLoopArgs(varset) 158 if ftype=='separate': loopVars['separate'].append(keynm) 159 160 # Now check args 161 if 'args' not in testDict: return loopVars 162 for varset in re.split('(^|\W)-(?=[a-zA-Z])',testDict['args']): 163 if not varset.strip(): continue 164 if '{{' in varset: 165 # Assuming only one for loop per var specification 166 keynm,lvars,ftype=parseLoopArgs(varset) 167 loopVars[ftype].append(keynm) 168 169 return loopVars 170 171def _getNewArgs(args,separate=True): 172 """ 173 Given: String that has args that might have loops in them 174 Return: All of the arguments/values that do not have 175 for 'separate output' in for loops 176 unless separate=False 177 """ 178 newargs='' 179 if not args.strip(): return args 180 for varset in re.split('(^|\W)-(?=[a-zA-Z])',args): 181 if not varset.strip(): continue 182 if '{{' in varset: 183 if separate: 184 if 'separate' in varset: continue 185 else: 186 if 'separate' not in varset: continue 187 188 newargs+="-"+varset.strip()+" " 189 190 return newargs 191 192def _getVarVals(findvar,testDict): 193 """ 194 Given: variable that is either nsize or in args 195 Return: Values to loop over and the other arguments 196 Note that we keep the other arguments even if they have 197 for loops to enable stepping through all of the for lops 198 """ 199 save_vals=None 200 if findvar=='nsize': 201 varset=testDict[findvar] 202 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 203 else: 204 varlist=[] 205 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 206 if not varset.strip(): continue 207 if '{{' not in varset: continue 208 keyvar,vals,ftype=parseLoopArgs(varset) 209 if keyvar==findvar: 210 save_vals=vals 211 212 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 213 return save_vals 214 215def genTestsSeparateTestvars(intests,indicts,final=False): 216 """ 217 Given: testname, sdict with 'separate_testvars 218 Return: testnames,sdicts: List of generated tests 219 The tricky part here is the {{ ... }separate output} 220 that can be used multiple times 221 """ 222 testnames=[]; sdicts=[] 223 for i in range(len(intests)): 224 testname=intests[i]; sdict=indicts[i]; i+=1 225 loopVars=_getLoopVars(sdict) 226 if len(loopVars['shared'])>0 and not final: 227 # Need to remove shared loop vars and push down to subtests 228 if 'subtests' in sdict: 229 for varset in re.split('(^|\W)-(?=[a-zA-Z])',sdict['args']): 230 if '{{' in varset: 231 for stest in sdict['subtests']: 232 if 'args' in sdict[stest]: 233 sdict[stest]['args']+=' -'+varset 234 else: 235 sdict[stest]['args']="-"+varset 236 sdict['args']=_getNewArgs(sdict['args'],separate=False) 237 if len(loopVars['separate'])>0: 238 sep_dicts=[sdict.copy()] 239 if 'args' in sep_dicts[0]: 240 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 241 sep_testnames=[testname] 242 for kvar in loopVars['separate']: 243 kvals=_getVarVals(kvar,sdict) 244 245 # Have to do loop over previous var/val combos as well 246 # and accumulate as we go 247 val_testnames=[]; val_dicts=[] 248 for val in kvals.split(): 249 gensuffix="_"+kvar+"-"+val.replace(',','__') 250 for kvaltestnm in sep_testnames: 251 val_testnames.append(kvaltestnm+gensuffix) 252 for kv in sep_dicts: 253 kvardict=kv.copy() 254 # If the last var then we have the final version 255 if 'suffix' in sdict: 256 kvardict['suffix']+=gensuffix 257 else: 258 kvardict['suffix']=gensuffix 259 if kvar=='nsize': 260 kvardict[kvar]=val 261 else: 262 kvardict['args']+="-"+kvar+" "+val+" " 263 val_dicts.append(kvardict) 264 sep_testnames=val_testnames 265 sep_dicts=val_dicts 266 testnames+=sep_testnames 267 sdicts+=sep_dicts 268 else: 269 # These are plain vanilla tests (no subtests, no loops) that 270 # do not have a suffix. This makes the targets match up with 271 # the output file (testname_1.out) 272 if final: 273 if '_' not in testname: testname+='_1' 274 testnames.append(testname) 275 sdicts.append(sdict) 276 return testnames,sdicts 277 278def genTestsSubtestSuffix(testnames,sdicts): 279 """ 280 Given: testname, sdict with separate_testvars 281 Return: testnames,sdicts: List of generated tests 282 """ 283 tnms=[]; sdcts=[] 284 for i in range(len(testnames)): 285 testname=testnames[i] 286 rmsubtests=[]; keepSubtests=False 287 if 'subtests' in sdicts[i]: 288 for stest in sdicts[i]["subtests"]: 289 if 'suffix' in sdicts[i][stest]: 290 rmsubtests.append(stest) 291 gensuffix="_"+sdicts[i][stest]['suffix'] 292 newtestnm=testname+gensuffix 293 tnms.append(newtestnm) 294 newsdict=sdicts[i].copy() 295 del newsdict['subtests'] 296 # Have to hand update 297 # Append 298 for kup in appendlist: 299 if kup in sdicts[i][stest]: 300 if kup in sdicts[i]: 301 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 302 else: 303 newsdict[kup]=sdicts[i][stest][kup] 304 # Promote 305 for kup in acceptedkeys: 306 if kup in appendlist: continue 307 if kup in sdicts[i][stest]: 308 newsdict[kup]=sdicts[i][stest][kup] 309 # Cleanup 310 for st in sdicts[i]["subtests"]: del newsdict[st] 311 sdcts.append(newsdict) 312 else: 313 keepSubtests=True 314 else: 315 tnms.append(testnames[i]) 316 sdcts.append(sdicts[i]) 317 # If a subtest without a suffix exists, then save it 318 if keepSubtests: 319 tnms.append(testnames[i]) 320 newsdict=sdicts[i].copy() 321 # Prune the tests to prepare for keeping 322 for rmtest in rmsubtests: 323 newsdict['subtests'].remove(rmtest) 324 del newsdict[rmtest] 325 sdcts.append(newsdict) 326 i+=1 327 return tnms,sdcts 328 329def splitTests(testname,sdict): 330 """ 331 Given: testname and dictionary generated from the YAML-like definition 332 Return: list of names and dictionaries corresponding to each test 333 given that the YAML-like language allows for multiple tests 334 """ 335 336 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 337 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 338 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 339 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 340 341 # Because I am altering the list, I do this in passes. Inelegant 342 343 return testnames, sdicts 344 345 346def testSplit(striptest): 347 """ 348 Split up a test into lines, but use a shell parser to detect when newlines are within quotation marks 349 and keep those together 350 """ 351 import shlex 352 353 sl = shlex.shlex() 354 sl.whitespace_split = True # only split at whitespace 355 sl.commenters = '' 356 sl.push_source(striptest) 357 last_pos = sl.instream.tell() 358 try: 359 last_token = sl.read_token() 360 except ValueError: 361 print(striptest) 362 raise ValueError 363 last_line = '' 364 while last_token != '': 365 new_pos = sl.instream.tell() 366 block = striptest[last_pos:new_pos] 367 token_start = block.find(last_token) 368 leading = block[0:token_start] 369 trailing = block[(token_start + len(last_token)):] 370 leading_split = leading.split('\n') 371 if len(leading_split) > 1: 372 yield last_line 373 last_line = '' 374 last_line += leading_split[-1] 375 last_line += last_token 376 trailing_split = trailing.split('\n') 377 last_line += trailing_split[0] 378 if len(trailing_split) > 1: 379 yield last_line 380 last_line = '' 381 last_pos = new_pos 382 try: 383 last_token = sl.read_token() 384 except ValueError: 385 print(striptest) 386 raise ValueError 387 yield last_line 388 389 390def parseTest(testStr,srcfile,verbosity): 391 """ 392 This parses an individual test 393 Our YAML-like language is hierarchial so should use a state machine in the general case, 394 but in practice we only support two levels of test: 395 """ 396 basename=os.path.basename(srcfile) 397 # Handle the new at the begininng 398 bn=re.sub("new_","",basename) 399 # This is the default 400 testname="run"+os.path.splitext(bn)[0] 401 402 # Tests that have default everything (so empty effectively) 403 if len(testStr)==0: 404 if '_' not in testname: testname+='_1' 405 return [testname], [{}] 406 407 striptest=_stripIndent(testStr,srcfile) 408 409 # go through and parse 410 subtestnum=0 411 subdict={} 412 comments=[] 413 indentlevel=0 414 for ln in testSplit(striptest): 415 line=ln.split('#')[0].rstrip() 416 if verbosity>2: print(line) 417 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 418 if comment: comments.append(comment) 419 if not line.strip(): continue 420 lsplit=line.split(':') 421 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 422 indentcount=lsplit[0].count(" ") 423 var=lsplit[0].strip() 424 val=line[line.find(':')+1:].strip() 425 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 426 # Start by seeing if we are in a subtest 427 if line.startswith(" "): 428 if var in subdict[subtestname]: 429 subdict[subtestname][var]+=" "+val 430 else: 431 subdict[subtestname][var]=val 432 if not indentlevel: indentlevel=indentcount 433 #if indentlevel!=indentcount: print("Error in indentation:", ln) 434 # Determine subtest name and make dict 435 elif var=="test": 436 subtestname="test"+str(subtestnum) 437 subdict[subtestname]={} 438 if "subtests" not in subdict: subdict["subtests"]=[] 439 subdict["subtests"].append(subtestname) 440 subtestnum=subtestnum+1 441 # The rest are easy 442 else: 443 # For convenience, it is sometimes convenient to list twice 444 if var in subdict: 445 if var in appendlist: 446 subdict[var]+=" "+val 447 else: 448 raise Exception(var+" entered twice: "+line) 449 else: 450 subdict[var]=val 451 if var=="suffix": 452 if len(val)>0: 453 testname+="_"+val 454 455 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 456 457 # A test block can create multiple tests. This does that logic 458 testnames,subdicts=splitTests(testname,subdict) 459 return testnames,subdicts 460 461def parseTests(testStr,srcfile,fileNums,verbosity): 462 """ 463 Parse the YAML-like string describing tests and return 464 a dictionary with the info in the form of: 465 testDict[test][subtest] 466 """ 467 468 testDict={} 469 470 # The first entry should be test: but it might be indented. 471 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 472 if verbosity>2: print(srcfile) 473 474 ## Check and see if we have build requirements 475 addToRunRequirements=None 476 if "\nbuild:" in newTestStr: 477 testDict['build']={} 478 # The file info is already here and need to append 479 Part1=newTestStr.split("build:")[1] 480 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 481 for bkey in buildkeys: 482 if bkey+":" in fileInfo: 483 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 484 #if verbosity>1: bkey+": "+testDict['build'][bkey] 485 # If a runtime requires are put into build, push them down to all run tests 486 # At this point, we are working with strings and not lists 487 if 'requires' in testDict['build']: 488 addToRunRequirements=testDict['build']['requires'] 489 # People put datafilespath into build, but it needs to be a runtime 490 if 'datafilespath' in testDict['build']['requires']: 491 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 492 testDict['build']['requires']=newreqs.strip() 493 494 495 # Now go through each test. First elem in split is blank 496 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 497 testnames,subdicts=parseTest(test,srcfile,verbosity) 498 for i in range(len(testnames)): 499 if testnames[i] in testDict: 500 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 501 # Add in build requirements that need to be moved 502 if addToRunRequirements: 503 if 'requires' in subdicts[i]: 504 subdicts[i]['requires']+=' '+addToRunRequirements 505 else: 506 subdicts[i]['requires']=addToRunRequirements 507 testDict[testnames[i]]=subdicts[i] 508 509 return testDict 510 511def parseTestFile(srcfile,verbosity): 512 """ 513 Parse single example files and return dictionary of the form: 514 testDict[srcfile][test][subtest] 515 """ 516 debug=False 517 basename=os.path.basename(srcfile) 518 if basename=='makefile': return {} 519 520 curdir=os.path.realpath(os.path.curdir) 521 basedir=os.path.dirname(os.path.realpath(srcfile)) 522 os.chdir(basedir) 523 524 testDict={} 525 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 526 527 ## Start with doing the tests 528 # 529 fsplit=fileStr.split("/*TEST\n")[1:] 530 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 531 # Allow for multiple "/*TEST" blocks even though it really should be 532 # one 533 srcTests=[] 534 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 535 testString=" ".join(srcTests) 536 flen=len(testString.split("\n")) 537 fend=fstart+flen-1 538 fileNums=range(fstart,fend) 539 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 540 # Massage dictionary for build requirements 541 if 'build' in testDict[basename]: 542 testDict[basename].update(testDict[basename]['build']) 543 del testDict[basename]['build'] 544 545 546 os.chdir(curdir) 547 return testDict 548 549def parseTestDir(directory,verbosity): 550 """ 551 Parse single example files and return dictionary of the form: 552 testDict[srcfile][test][subtest] 553 """ 554 curdir=os.path.realpath(os.path.curdir) 555 basedir=os.path.realpath(directory) 556 os.chdir(basedir) 557 558 tDict={} 559 for test_file in sorted(glob.glob("new_ex*.*")): 560 tDict.update(parseTestFile(test_file,verbosity)) 561 562 os.chdir(curdir) 563 return tDict 564 565def printExParseDict(rDict): 566 """ 567 This is useful for debugging 568 """ 569 indent=" " 570 for sfile in rDict: 571 print(sfile) 572 sortkeys=list(rDict[sfile].keys()) 573 sortkeys.sort() 574 for runex in sortkeys: 575 if runex == 'requires': 576 print(indent+runex+':'+str(rDict[sfile][runex])) 577 continue 578 print(indent+runex) 579 if type(rDict[sfile][runex])==bytes: 580 print(indent*2+rDict[sfile][runex]) 581 else: 582 for var in rDict[sfile][runex]: 583 if var.startswith("test"): continue 584 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 585 if 'subtests' in rDict[sfile][runex]: 586 for var in rDict[sfile][runex]['subtests']: 587 print(indent*2+var) 588 for var2 in rDict[sfile][runex][var]: 589 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 590 print("\n") 591 return 592 593def main(directory='',test_file='',verbosity=0): 594 595 if directory: 596 tDict=parseTestDir(directory,verbosity) 597 else: 598 tDict=parseTestFile(test_file,verbosity) 599 if verbosity>0: printExParseDict(tDict) 600 601 return 602 603if __name__ == '__main__': 604 import optparse 605 parser = optparse.OptionParser() 606 parser.add_option('-d', '--directory', dest='directory', 607 default="", help='Directory containing files to parse') 608 parser.add_option('-t', '--test_file', dest='test_file', 609 default="", help='Test file, e.g., ex1.c, to parse') 610 parser.add_option('-v', '--verbosity', dest='verbosity', 611 help='Verbosity of output by level: 1, 2, or 3', default=0) 612 opts, extra_args = parser.parse_args() 613 614 if extra_args: 615 import sys 616 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 617 exit(1) 618 if not opts.test_file and not opts.directory: 619 print("test file or directory is required") 620 parser.print_usage() 621 sys.exit() 622 623 # Need verbosity to be an integer 624 try: 625 verbosity=int(opts.verbosity) 626 except: 627 raise Exception("Error: Verbosity must be integer") 628 629 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 630