1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix diff_args args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args diff_args requires comments".split() 54 55import re 56 57def getDefaultOutputFileRoot(testname): 58 """ 59 Given testname, give DefaultRoot and DefaultOutputFilename 60 e.g., runex1 gives ex1_1, output/ex1_1.out 61 """ 62 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 63 if not "_" in defroot: defroot=defroot+"_1" 64 return defroot 65 66def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 67 """ 68 Go through and remove a level of indentation 69 Also strip of trailing whitespace 70 """ 71 # The first entry should be test: but it might be indented. 72 ext=os.path.splitext(srcfile)[1] 73 stripstr=" " 74 if len(fileNums)>0: lineNum=fileNums[0]-1 75 for lline in block.split("\n"): 76 if len(fileNums)>0: lineNum+=1 77 line=lline[1:] if lline.startswith("!") else lline 78 if not line.strip(): continue 79 if line.strip().startswith('#'): continue 80 if entireBlock: 81 var=line.split(":")[0].strip() 82 if not var in ['test','testset','build']: 83 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 84 nspace=len(line)-len(line.lstrip(stripstr)) 85 newline=line[nspace:] 86 break 87 88 # Strip off any indentation for the whole string and any trailing 89 # whitespace for convenience 90 newTestStr="\n" 91 if len(fileNums)>0: lineNum=fileNums[0]-1 92 firstPass=True 93 for lline in block.split("\n"): 94 if len(fileNums)>0: lineNum+=1 95 line=lline[1:] if lline.startswith("!") else lline 96 if not line.strip(): continue 97 if line.strip().startswith('#'): 98 newTestStr+=line+'\n' 99 else: 100 newline=line[nspace:] 101 newTestStr+=newline.rstrip()+"\n" 102 # Do some basic indentation checks 103 if entireBlock: 104 # Don't need to check comment lines 105 if line.strip().startswith('#'): continue 106 if not newline.startswith(" "): 107 var=newline.split(":")[0].strip() 108 if not var in ['test','testset','build']: 109 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 110 if len(fileNums)>0: 111 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 112 else: 113 raise Exception(err) 114 else: 115 var=line.split(":")[0].strip() 116 if var in ['test','testset','build']: 117 subnspace=len(line)-len(line.lstrip(stripstr)) 118 if firstPass: 119 firstsubnspace=subnspace 120 firstPass=False 121 else: 122 if firstsubnspace!=subnspace: 123 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 124 if len(fileNums)>0: 125 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 126 else: 127 raise Exception(err) 128 129 # Allow line continuation character '\' 130 return newTestStr.replace('\\\n', ' ') 131 132def parseLoopArgs(varset): 133 """ 134 Given: String containing loop variables 135 Return: tuple containing separate/shared and string of loop vars 136 """ 137 keynm=varset.split("{{")[0].strip().lstrip('-') 138 if not keynm.strip(): keynm='nsize' 139 lvars=varset.split('{{')[1].split('}')[0] 140 suffx=varset.split('{{')[1].split('}')[1] 141 ftype='separate' if suffx.startswith('separate') else 'shared' 142 return keynm,lvars,ftype 143 144def _getLoopVars(testDict): 145 """ 146 Given: dictionary that may have 147 Return: Variables that cause a test split 148 """ 149 vals=None 150 loopVars={} 151 loopVars['separate']=[] 152 loopVars['shared']=[] 153 # Check nsize 154 if 'nsize' in testDict: 155 varset=testDict['nsize'] 156 if '{{' in varset: 157 keynm,lvars,ftype=parseLoopArgs(varset) 158 if ftype=='separate': loopVars['separate'].append(keynm) 159 160 # Now check args 161 if 'args' not in testDict: return loopVars 162 for varset in re.split('(^|\W)-(?=[a-zA-Z])',testDict['args']): 163 if not varset.strip(): continue 164 if '{{' in varset: 165 # Assuming only one for loop per var specification 166 keynm,lvars,ftype=parseLoopArgs(varset) 167 loopVars[ftype].append(keynm) 168 169 return loopVars 170 171def _getNewArgs(args,separate=True): 172 """ 173 Given: String that has args that might have loops in them 174 Return: All of the arguments/values that do not have 175 for 'separate output' in for loops 176 unless separate=False 177 """ 178 newargs='' 179 if not args.strip(): return args 180 for varset in re.split('(^|\W)-(?=[a-zA-Z])',args): 181 if not varset.strip(): continue 182 if '{{' in varset: 183 if separate: 184 if 'separate' in varset: continue 185 else: 186 if 'separate' not in varset: continue 187 188 newargs+="-"+varset.strip()+" " 189 190 return newargs 191 192def _getVarVals(findvar,testDict): 193 """ 194 Given: variable that is either nsize or in args 195 Return: Values to loop over and the other arguments 196 Note that we keep the other arguments even if they have 197 for loops to enable stepping through all of the for lops 198 """ 199 save_vals=None 200 if findvar=='nsize': 201 varset=testDict[findvar] 202 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 203 else: 204 varlist=[] 205 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 206 if not varset.strip(): continue 207 if '{{' not in varset: continue 208 keyvar,vals,ftype=parseLoopArgs(varset) 209 if keyvar==findvar: 210 save_vals=vals 211 212 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 213 return save_vals 214 215def genTestsSeparateTestvars(intests,indicts,final=False): 216 """ 217 Given: testname, sdict with 'separate_testvars 218 Return: testnames,sdicts: List of generated tests 219 The tricky part here is the {{ ... }separate output} 220 that can be used multiple times 221 """ 222 testnames=[]; sdicts=[] 223 for i in range(len(intests)): 224 testname=intests[i]; sdict=indicts[i]; i+=1 225 loopVars=_getLoopVars(sdict) 226 if len(loopVars['shared'])>0 and not final: 227 # Need to remove shared loop vars and push down to subtests 228 if 'subtests' in sdict: 229 for varset in re.split('(^|\W)-(?=[a-zA-Z])',sdict['args']): 230 if '{{' in varset: 231 for stest in sdict['subtests']: 232 if 'args' in sdict[stest]: 233 sdict[stest]['args']+=' -'+varset 234 else: 235 sdict[stest]['args']="-"+varset 236 sdict['args']=_getNewArgs(sdict['args'],separate=False) 237 if len(loopVars['separate'])>0: 238 sep_dicts=[sdict.copy()] 239 if 'args' in sep_dicts[0]: 240 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 241 sep_testnames=[testname] 242 for kvar in loopVars['separate']: 243 kvals=_getVarVals(kvar,sdict) 244 245 # Have to do loop over previous var/val combos as well 246 # and accumulate as we go 247 val_testnames=[]; val_dicts=[] 248 for val in kvals.split(): 249 gensuffix="_"+kvar+"-"+val.replace(',','__') 250 for kvaltestnm in sep_testnames: 251 val_testnames.append(kvaltestnm+gensuffix) 252 for kv in sep_dicts: 253 kvardict=kv.copy() 254 # If the last var then we have the final version 255 if 'suffix' in sdict: 256 kvardict['suffix']+=gensuffix 257 else: 258 kvardict['suffix']=gensuffix 259 if kvar=='nsize': 260 kvardict[kvar]=val 261 else: 262 kvardict['args']+="-"+kvar+" "+val+" " 263 val_dicts.append(kvardict) 264 sep_testnames=val_testnames 265 sep_dicts=val_dicts 266 testnames+=sep_testnames 267 sdicts+=sep_dicts 268 else: 269 # These are plain vanilla tests (no subtests, no loops) that 270 # do not have a suffix. This makes the targets match up with 271 # the output file (testname_1.out) 272 if final: 273 if '_' not in testname: testname+='_1' 274 testnames.append(testname) 275 sdicts.append(sdict) 276 return testnames,sdicts 277 278def genTestsSubtestSuffix(testnames,sdicts): 279 """ 280 Given: testname, sdict with separate_testvars 281 Return: testnames,sdicts: List of generated tests 282 """ 283 tnms=[]; sdcts=[] 284 for i in range(len(testnames)): 285 testname=testnames[i] 286 rmsubtests=[]; keepSubtests=False 287 if 'subtests' in sdicts[i]: 288 for stest in sdicts[i]["subtests"]: 289 if 'suffix' in sdicts[i][stest]: 290 rmsubtests.append(stest) 291 gensuffix="_"+sdicts[i][stest]['suffix'] 292 newtestnm=testname+gensuffix 293 tnms.append(newtestnm) 294 newsdict=sdicts[i].copy() 295 del newsdict['subtests'] 296 # Have to hand update 297 # Append 298 for kup in appendlist: 299 if kup in sdicts[i][stest]: 300 if kup in sdicts[i]: 301 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 302 else: 303 newsdict[kup]=sdicts[i][stest][kup] 304 # Promote 305 for kup in acceptedkeys: 306 if kup in appendlist: continue 307 if kup in sdicts[i][stest]: 308 newsdict[kup]=sdicts[i][stest][kup] 309 # Cleanup 310 for st in sdicts[i]["subtests"]: del newsdict[st] 311 sdcts.append(newsdict) 312 else: 313 keepSubtests=True 314 else: 315 tnms.append(testnames[i]) 316 sdcts.append(sdicts[i]) 317 # If a subtest without a suffix exists, then save it 318 if keepSubtests: 319 tnms.append(testnames[i]) 320 newsdict=sdicts[i].copy() 321 # Prune the tests to prepare for keeping 322 for rmtest in rmsubtests: 323 newsdict['subtests'].remove(rmtest) 324 del newsdict[rmtest] 325 sdcts.append(newsdict) 326 i+=1 327 return tnms,sdcts 328 329def splitTests(testname,sdict): 330 """ 331 Given: testname and YAML-generated dictionary 332 Return: list of names and dictionaries corresponding to each test 333 given that the YAML language allows for multiple tests 334 """ 335 336 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 337 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 338 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 339 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 340 341 # Because I am altering the list, I do this in passes. Inelegant 342 343 return testnames, sdicts 344 345def parseTest(testStr,srcfile,verbosity): 346 """ 347 This parses an individual test 348 YAML is hierarchial so should use a state machine in the general case, 349 but in practice we only support two levels of test: 350 """ 351 basename=os.path.basename(srcfile) 352 # Handle the new at the begininng 353 bn=re.sub("new_","",basename) 354 # This is the default 355 testname="run"+os.path.splitext(bn)[0] 356 357 # Tests that have default everything (so empty effectively) 358 if len(testStr)==0: 359 if '_' not in testname: testname+='_1' 360 return [testname], [{}] 361 362 striptest=_stripIndent(testStr,srcfile) 363 364 # go through and parse 365 subtestnum=0 366 subdict={} 367 comments=[] 368 indentlevel=0 369 for ln in striptest.split("\n"): 370 line=ln.split('#')[0].rstrip() 371 if verbosity>2: print(line) 372 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 373 if comment: comments.append(comment) 374 if not line.strip(): continue 375 lsplit=line.split(':') 376 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 377 indentcount=lsplit[0].count(" ") 378 var=lsplit[0].strip() 379 val=line[line.find(':')+1:].strip() 380 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 381 # Start by seeing if we are in a subtest 382 if line.startswith(" "): 383 if var in subdict[subtestname]: 384 subdict[subtestname][var]+=" "+val 385 else: 386 subdict[subtestname][var]=val 387 if not indentlevel: indentlevel=indentcount 388 #if indentlevel!=indentcount: print("Error in indentation:", ln) 389 # Determine subtest name and make dict 390 elif var=="test": 391 subtestname="test"+str(subtestnum) 392 subdict[subtestname]={} 393 if "subtests" not in subdict: subdict["subtests"]=[] 394 subdict["subtests"].append(subtestname) 395 subtestnum=subtestnum+1 396 # The rest are easy 397 else: 398 # For convenience, it is sometimes convenient to list twice 399 if var in subdict: 400 if var in appendlist: 401 subdict[var]+=" "+val 402 else: 403 raise Exception(var+" entered twice: "+line) 404 else: 405 subdict[var]=val 406 if var=="suffix": 407 if len(val)>0: 408 testname+="_"+val 409 410 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 411 412 # A test block can create multiple tests. This does that logic 413 testnames,subdicts=splitTests(testname,subdict) 414 return testnames,subdicts 415 416def parseTests(testStr,srcfile,fileNums,verbosity): 417 """ 418 Parse the yaml string describing tests and return 419 a dictionary with the info in the form of: 420 testDict[test][subtest] 421 This is an inelegant parser as we do not wish to 422 introduce a new dependency by bringing in pyyaml. 423 The advantage is that validation can be done as 424 it is parsed (e.g., 'test' is the only top-level node) 425 """ 426 427 testDict={} 428 429 # The first entry should be test: but it might be indented. 430 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 431 if verbosity>2: print(srcfile) 432 433 ## Check and see if we have build requirements 434 addToRunRequirements=None 435 if "\nbuild:" in newTestStr: 436 testDict['build']={} 437 # The file info is already here and need to append 438 Part1=newTestStr.split("build:")[1] 439 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 440 for bkey in buildkeys: 441 if bkey+":" in fileInfo: 442 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 443 #if verbosity>1: bkey+": "+testDict['build'][bkey] 444 # If a runtime requires are put into build, push them down to all run tests 445 # At this point, we are working with strings and not lists 446 if 'requires' in testDict['build']: 447 addToRunRequirements=testDict['build']['requires'] 448 # People put datafilespath into build, but it needs to be a runtime 449 if 'datafilespath' in testDict['build']['requires']: 450 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 451 testDict['build']['requires']=newreqs.strip() 452 453 454 # Now go through each test. First elem in split is blank 455 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 456 testnames,subdicts=parseTest(test,srcfile,verbosity) 457 for i in range(len(testnames)): 458 if testnames[i] in testDict: 459 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 460 # Add in build requirements that need to be moved 461 if addToRunRequirements: 462 if 'requires' in subdicts[i]: 463 subdicts[i]['requires']+=' '+addToRunRequirements 464 else: 465 subdicts[i]['requires']=addToRunRequirements 466 testDict[testnames[i]]=subdicts[i] 467 468 return testDict 469 470def parseTestFile(srcfile,verbosity): 471 """ 472 Parse single example files and return dictionary of the form: 473 testDict[srcfile][test][subtest] 474 """ 475 debug=False 476 basename=os.path.basename(srcfile) 477 if basename=='makefile': return {} 478 479 curdir=os.path.realpath(os.path.curdir) 480 basedir=os.path.dirname(os.path.realpath(srcfile)) 481 os.chdir(basedir) 482 483 testDict={} 484 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 485 486 ## Start with doing the tests 487 # 488 fsplit=fileStr.split("/*TEST\n")[1:] 489 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 490 # Allow for multiple "/*TEST" blocks even though it really should be 491 # one 492 srcTests=[] 493 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 494 testString=" ".join(srcTests) 495 flen=len(testString.split("\n")) 496 fend=fstart+flen-1 497 fileNums=range(fstart,fend) 498 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 499 # Massage dictionary for build requirements 500 if 'build' in testDict[basename]: 501 testDict[basename].update(testDict[basename]['build']) 502 del testDict[basename]['build'] 503 504 505 os.chdir(curdir) 506 return testDict 507 508def parseTestDir(directory,verbosity): 509 """ 510 Parse single example files and return dictionary of the form: 511 testDict[srcfile][test][subtest] 512 """ 513 curdir=os.path.realpath(os.path.curdir) 514 basedir=os.path.realpath(directory) 515 os.chdir(basedir) 516 517 tDict={} 518 for test_file in sorted(glob.glob("new_ex*.*")): 519 tDict.update(parseTestFile(test_file,verbosity)) 520 521 os.chdir(curdir) 522 return tDict 523 524def printExParseDict(rDict): 525 """ 526 This is useful for debugging 527 """ 528 indent=" " 529 for sfile in rDict: 530 print(sfile) 531 sortkeys=list(rDict[sfile].keys()) 532 sortkeys.sort() 533 for runex in sortkeys: 534 if runex == 'requires': 535 print(indent+runex+':'+str(rDict[sfile][runex])) 536 continue 537 print(indent+runex) 538 if type(rDict[sfile][runex])==bytes: 539 print(indent*2+rDict[sfile][runex]) 540 else: 541 for var in rDict[sfile][runex]: 542 if var.startswith("test"): continue 543 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 544 if 'subtests' in rDict[sfile][runex]: 545 for var in rDict[sfile][runex]['subtests']: 546 print(indent*2+var) 547 for var2 in rDict[sfile][runex][var]: 548 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 549 print("\n") 550 return 551 552def main(directory='',test_file='',verbosity=0): 553 554 if directory: 555 tDict=parseTestDir(directory,verbosity) 556 else: 557 tDict=parseTestFile(test_file,verbosity) 558 if verbosity>0: printExParseDict(tDict) 559 560 return 561 562if __name__ == '__main__': 563 import optparse 564 parser = optparse.OptionParser() 565 parser.add_option('-d', '--directory', dest='directory', 566 default="", help='Directory containing files to parse') 567 parser.add_option('-t', '--test_file', dest='test_file', 568 default="", help='Test file, e.g., ex1.c, to parse') 569 parser.add_option('-v', '--verbosity', dest='verbosity', 570 help='Verbosity of output by level: 1, 2, or 3', default=0) 571 opts, extra_args = parser.parse_args() 572 573 if extra_args: 574 import sys 575 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 576 exit(1) 577 if not opts.test_file and not opts.directory: 578 print("test file or directory is required") 579 parser.print_usage() 580 sys.exit() 581 582 # Need verbosity to be an integer 583 try: 584 verbosity=int(opts.verbosity) 585 except: 586 raise Exception("Error: Verbosity must be integer") 587 588 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 589