1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37 38import os, re, glob, types 39import sys 40import logging 41sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 42 43import inspect 44thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 45maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 46sys.path.insert(0,maintdir) 47 48# These are special keys describing build 49buildkeys="requires TODO SKIP depends".split() 50 51acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 52appendlist="args requires comments".split() 53 54import re 55 56def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 57 """ 58 Go through and remove a level of indentation 59 Also strip of trailing whitespace 60 """ 61 # The first entry should be test: but it might be indented. 62 ext=os.path.splitext(srcfile)[1] 63 stripstr=" " 64 if len(fileNums)>0: lineNum=fileNums[0]-1 65 for lline in block.split("\n"): 66 if len(fileNums)>0: lineNum+=1 67 line=lline[1:] if lline.startswith("!") else lline 68 if not line.strip(): continue 69 if line.strip().startswith('#'): continue 70 if entireBlock: 71 var=line.split(":")[0].strip() 72 if not var in ['test','testset','build']: 73 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 74 nspace=len(line)-len(line.lstrip(stripstr)) 75 newline=line[nspace:] 76 break 77 78 # Strip off any indentation for the whole string and any trailing 79 # whitespace for convenience 80 newTestStr="\n" 81 if len(fileNums)>0: lineNum=fileNums[0]-1 82 firstPass=True 83 for lline in block.split("\n"): 84 if len(fileNums)>0: lineNum+=1 85 line=lline[1:] if lline.startswith("!") else lline 86 if not line.strip(): continue 87 if line.strip().startswith('#'): 88 newTestStr+=line+'\n' 89 else: 90 newline=line[nspace:] 91 newTestStr+=newline.rstrip()+"\n" 92 # Do some basic indentation checks 93 if entireBlock: 94 # Don't need to check comment lines 95 if line.strip().startswith('#'): continue 96 if not newline.startswith(" "): 97 var=newline.split(":")[0].strip() 98 if not var in ['test','testset','build']: 99 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 100 if len(fileNums)>0: 101 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 102 else: 103 raise Exception(err) 104 else: 105 var=line.split(":")[0].strip() 106 if var in ['test','testset','build']: 107 subnspace=len(line)-len(line.lstrip(stripstr)) 108 if firstPass: 109 firstsubnspace=subnspace 110 firstPass=False 111 else: 112 if firstsubnspace!=subnspace: 113 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 114 if len(fileNums)>0: 115 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 116 else: 117 raise Exception(err) 118 119 # Allow line continuation character '\' 120 return newTestStr.replace('\\\n', ' ') 121 122def parseLoopArgs(varset): 123 """ 124 Given: String containing loop variables 125 Return: tuple containing separate/shared and string of loop vars 126 """ 127 keynm=varset.split("{{")[0].strip() 128 if not keynm.strip(): keynm='nsize' 129 lvars=varset.split('{{')[1].split('}')[0] 130 suffx=varset.split('{{')[1].split('}')[1] 131 ftype='separate' if suffx.startswith('separate') else 'shared' 132 return keynm,lvars,ftype 133 134def _getSeparateTestvars(testDict): 135 """ 136 Given: dictionary that may have 137 Return: Variables that cause a test split 138 """ 139 vals=None 140 sepvars=[] 141 # Check nsize 142 if testDict.has_key('nsize'): 143 varset=testDict['nsize'] 144 if '{{' in varset: 145 keynm,lvars,ftype=parseLoopArgs(varset) 146 if ftype=='separate': sepvars.append(keynm) 147 148 # Now check args 149 if not testDict.has_key('args'): return sepvars 150 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 151 if not varset.strip(): continue 152 if '{{' in varset: 153 # Assuming only one for loop per var specification 154 keynm,lvars,ftype=parseLoopArgs(varset) 155 if ftype=='separate': sepvars.append(keynm) 156 157 return sepvars 158 159def _getNewArgs(args): 160 """ 161 Given: String that has args that might have loops in them 162 Return: All of the arguments/values that do not have 163 for 'separate output' in for loops 164 """ 165 newargs='' 166 if not args.strip(): return args 167 for varset in re.split('-(?=[a-zA-Z])',args): 168 if not varset.strip(): continue 169 if '{{' not in varset: 170 if 'separate' not in varset: 171 newargs+="-"+varset.strip()+" " 172 173 return newargs 174 175def _getVarVals(findvar,testDict): 176 """ 177 Given: variable that is either nsize or in args 178 Return: Values to loop over and the other arguments 179 Note that we keep the other arguments even if they have 180 for loops to enable stepping through all of the for lops 181 """ 182 save_vals=None 183 if findvar=='nsize': 184 varset=testDict[findvar] 185 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 186 else: 187 varlist=[] 188 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 189 if not varset.strip(): continue 190 if '{{' not in varset: continue 191 keyvar,vals,ftype=parseLoopArgs(varset) 192 if keyvar==findvar: 193 save_vals=vals 194 195 if not save_vals: raise StandardError("Could not find separate_testvar: "+findvar) 196 return save_vals 197 198def genTestsSeparateTestvars(intests,indicts): 199 """ 200 Given: testname, sdict with 'separate_testvars 201 Return: testnames,sdicts: List of generated tests 202 The tricky part here is the {{ ... }separate output} 203 that can be used multiple times 204 """ 205 testnames=[]; sdicts=[] 206 for i in range(len(intests)): 207 testname=intests[i]; sdict=indicts[i]; i+=1 208 separate_testvars=_getSeparateTestvars(sdict) 209 if len(separate_testvars)>0: 210 sep_dicts=[sdict.copy()] 211 if 'args' in sep_dicts[0]: 212 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 213 sep_testnames=[testname] 214 for kvar in separate_testvars: 215 kvals=_getVarVals(kvar,sdict) 216 217 # Have to do loop over previous var/val combos as well 218 # and accumulate as we go 219 val_testnames=[]; val_dicts=[] 220 for val in kvals.split(): 221 gensuffix="_"+kvar+"-"+val.replace(',','__') 222 for kvaltestnm in sep_testnames: 223 val_testnames.append(kvaltestnm+gensuffix) 224 for kv in sep_dicts: 225 kvardict=kv.copy() 226 # If the last var then we have the final version 227 if 'suffix' in sdict: 228 kvardict['suffix']+=gensuffix 229 else: 230 kvardict['suffix']=gensuffix 231 if kvar=='nsize': 232 kvardict[kvar]=val 233 else: 234 kvardict['args']+="-"+kvar+" "+val+" " 235 val_dicts.append(kvardict) 236 sep_testnames=val_testnames 237 sep_dicts=val_dicts 238 testnames+=sep_testnames 239 sdicts+=sep_dicts 240 else: 241 testnames.append(testname) 242 sdicts.append(sdict) 243 return testnames,sdicts 244 245def genTestsSubtestSuffix(testnames,sdicts): 246 """ 247 Given: testname, sdict with separate_testvars 248 Return: testnames,sdicts: List of generated tests 249 """ 250 tnms=[]; sdcts=[] 251 for i in range(len(testnames)): 252 testname=testnames[i] 253 rmsubtests=[]; keepSubtests=False 254 if sdicts[i].has_key('subtests'): 255 for stest in sdicts[i]["subtests"]: 256 if sdicts[i][stest].has_key('suffix'): 257 rmsubtests.append(stest) 258 gensuffix="_"+sdicts[i][stest]['suffix'] 259 newtestnm=testname+gensuffix 260 tnms.append(newtestnm) 261 newsdict=sdicts[i].copy() 262 del newsdict['subtests'] 263 # Have to hand update 264 # Append 265 for kup in appendlist: 266 if sdicts[i][stest].has_key(kup): 267 if sdicts[i].has_key(kup): 268 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 269 else: 270 newsdict[kup]=sdicts[i][stest][kup] 271 # Promote 272 for kup in acceptedkeys: 273 if kup in appendlist: continue 274 if sdicts[i][stest].has_key(kup): 275 newsdict[kup]=sdicts[i][stest][kup] 276 # Cleanup 277 for st in sdicts[i]["subtests"]: del newsdict[st] 278 sdcts.append(newsdict) 279 else: 280 keepSubtests=True 281 else: 282 tnms.append(testnames[i]) 283 sdcts.append(sdicts[i]) 284 # If a subtest without a suffix exists, then save it 285 if keepSubtests: 286 tnms.append(testnames[i]) 287 newsdict=sdicts[i].copy() 288 # Prune the tests to prepare for keeping 289 for rmtest in rmsubtests: 290 newsdict['subtests'].remove(rmtest) 291 del newsdict[rmtest] 292 sdcts.append(newsdict) 293 i+=1 294 return tnms,sdcts 295 296def splitTests(testname,sdict): 297 """ 298 Given: testname and YAML-generated dictionary 299 Return: list of names and dictionaries corresponding to each test 300 given that the YAML language allows for multiple tests 301 """ 302 303 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 304 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 305 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 306 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts) 307 308 # Because I am altering the list, I do this in passes. Inelegant 309 310 return testnames, sdicts 311 312def parseTest(testStr,srcfile,verbosity): 313 """ 314 This parses an individual test 315 YAML is hierarchial so should use a state machine in the general case, 316 but in practice we only support two levels of test: 317 """ 318 basename=os.path.basename(srcfile) 319 # Handle the new at the begininng 320 bn=re.sub("new_","",basename) 321 # This is the default 322 testname="run"+os.path.splitext(bn)[0] 323 324 # Tests that have default everything (so empty effectively) 325 if len(testStr)==0: return [testname], [{}] 326 327 striptest=_stripIndent(testStr,srcfile) 328 329 # go through and parse 330 subtestnum=0 331 subdict={} 332 comments=[] 333 indentlevel=0 334 for ln in striptest.split("\n"): 335 line=ln.split('#')[0].rstrip() 336 if verbosity>2: print(line) 337 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 338 if comment: comments.append(comment) 339 if not line.strip(): continue 340 lsplit=line.split(':') 341 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 342 indentcount=lsplit[0].count(" ") 343 var=lsplit[0].strip() 344 val=line[line.find(':')+1:].strip() 345 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 346 # Start by seeing if we are in a subtest 347 if line.startswith(" "): 348 subdict[subtestname][var]=val 349 if not indentlevel: indentlevel=indentcount 350 #if indentlevel!=indentcount: print("Error in indentation:", ln) 351 # Determine subtest name and make dict 352 elif var=="test": 353 subtestname="test"+str(subtestnum) 354 subdict[subtestname]={} 355 if not subdict.has_key("subtests"): subdict["subtests"]=[] 356 subdict["subtests"].append(subtestname) 357 subtestnum=subtestnum+1 358 # The rest are easy 359 else: 360 # For convenience, it is sometimes convenient to list twice 361 if subdict.has_key(var): 362 if var in appendlist: 363 subdict[var]+=" "+val 364 else: 365 raise Exception(var+" entered twice: "+line) 366 else: 367 subdict[var]=val 368 if var=="suffix": 369 if len(val)>0: 370 testname=testname+"_"+val 371 372 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 373 # A test block can create multiple tests. This does 374 # that logic 375 testnames,subdicts=splitTests(testname,subdict) 376 return testnames,subdicts 377 378def parseTests(testStr,srcfile,fileNums,verbosity): 379 """ 380 Parse the yaml string describing tests and return 381 a dictionary with the info in the form of: 382 testDict[test][subtest] 383 This is an inelegant parser as we do not wish to 384 introduce a new dependency by bringing in pyyaml. 385 The advantage is that validation can be done as 386 it is parsed (e.g., 'test' is the only top-level node) 387 """ 388 389 testDict={} 390 391 # The first entry should be test: but it might be indented. 392 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 393 if verbosity>2: print(srcfile) 394 395 ## Check and see if we have build requirements 396 addToRunRequirements=None 397 if "\nbuild:" in newTestStr: 398 testDict['build']={} 399 # The file info is already here and need to append 400 Part1=newTestStr.split("build:")[1] 401 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 402 for bkey in buildkeys: 403 if bkey+":" in fileInfo: 404 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 405 #if verbosity>1: bkey+": "+testDict['build'][bkey] 406 # If a runtime requires are put into build, push them down to all run tests 407 # At this point, we are working with strings and not lists 408 if 'requires' in testDict['build']: 409 if 'datafilespath' in testDict['build']['requires']: 410 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 411 testDict['build']['requires']=newreqs.strip() 412 addToRunRequirements='datafilespath' 413 414 415 # Now go through each test. First elem in split is blank 416 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 417 testnames,subdicts=parseTest(test,srcfile,verbosity) 418 for i in range(len(testnames)): 419 if testDict.has_key(testnames[i]): 420 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 421 # Add in build requirements that need to be moved 422 if addToRunRequirements: 423 if 'requires' in subdicts[i]: 424 subdicts[i]['requires']+=addToRunRequirements 425 else: 426 subdicts[i]['requires']=addToRunRequirements 427 testDict[testnames[i]]=subdicts[i] 428 429 return testDict 430 431def parseTestFile(srcfile,verbosity): 432 """ 433 Parse single example files and return dictionary of the form: 434 testDict[srcfile][test][subtest] 435 """ 436 debug=False 437 basename=os.path.basename(srcfile) 438 if basename=='makefile': return {} 439 440 curdir=os.path.realpath(os.path.curdir) 441 basedir=os.path.dirname(os.path.realpath(srcfile)) 442 os.chdir(basedir) 443 444 testDict={} 445 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 446 447 ## Start with doing the tests 448 # 449 fsplit=fileStr.split("/*TEST\n")[1:] 450 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 451 # Allow for multiple "/*TEST" blocks even though it really should be 452 # one 453 srcTests=[] 454 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 455 testString=" ".join(srcTests) 456 flen=len(testString.split("\n")) 457 fend=fstart+flen-1 458 fileNums=range(fstart,fend) 459 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 460 # Massage dictionary for build requirements 461 if 'build' in testDict[basename]: 462 testDict[basename].update(testDict[basename]['build']) 463 del testDict[basename]['build'] 464 465 466 os.chdir(curdir) 467 return testDict 468 469def parseTestDir(directory,verbosity): 470 """ 471 Parse single example files and return dictionary of the form: 472 testDict[srcfile][test][subtest] 473 """ 474 curdir=os.path.realpath(os.path.curdir) 475 basedir=os.path.realpath(directory) 476 os.chdir(basedir) 477 478 tDict={} 479 for test_file in glob.glob("new_ex*.*"): 480 tDict.update(parseTestFile(test_file,verbosity)) 481 482 os.chdir(curdir) 483 return tDict 484 485def printExParseDict(rDict): 486 """ 487 This is useful for debugging 488 """ 489 indent=" " 490 for sfile in rDict: 491 print(sfile) 492 sortkeys=rDict[sfile].keys() 493 sortkeys.sort() 494 for runex in sortkeys: 495 print(indent+runex) 496 if type(rDict[sfile][runex])==types.StringType: 497 print(indent*2+rDict[sfile][runex]) 498 else: 499 for var in rDict[sfile][runex]: 500 if var.startswith("test"): continue 501 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 502 if rDict[sfile][runex].has_key('subtests'): 503 for var in rDict[sfile][runex]['subtests']: 504 print(indent*2+var) 505 for var2 in rDict[sfile][runex][var]: 506 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 507 print("\n") 508 return 509 510def main(directory='',test_file='',verbosity=0): 511 512 if directory: 513 tDict=parseTestDir(directory,verbosity) 514 else: 515 tDict=parseTestFile(test_file,verbosity) 516 if verbosity>0: printExParseDict(tDict) 517 518 return 519 520if __name__ == '__main__': 521 import optparse 522 parser = optparse.OptionParser() 523 parser.add_option('-d', '--directory', dest='directory', 524 default="", help='Directory containing files to parse') 525 parser.add_option('-t', '--test_file', dest='test_file', 526 default="", help='Test file, e.g., ex1.c, to parse') 527 parser.add_option('-v', '--verbosity', dest='verbosity', 528 help='Verbosity of output by level: 1, 2, or 3', default=0) 529 opts, extra_args = parser.parse_args() 530 531 if extra_args: 532 import sys 533 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 534 exit(1) 535 if not opts.test_file and not opts.directory: 536 print("test file or directory is required") 537 parser.print_usage() 538 sys.exit() 539 540 # Need verbosity to be an integer 541 try: 542 verbosity=int(opts.verbosity) 543 except: 544 raise Exception("Error: Verbosity must be integer") 545 546 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 547