1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args requires comments".split() 54 55import re 56 57def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 58 """ 59 Go through and remove a level of indentation 60 Also strip of trailing whitespace 61 """ 62 # The first entry should be test: but it might be indented. 63 ext=os.path.splitext(srcfile)[1] 64 stripstr=" " 65 if len(fileNums)>0: lineNum=fileNums[0]-1 66 for lline in block.split("\n"): 67 if len(fileNums)>0: lineNum+=1 68 line=lline[1:] if lline.startswith("!") else lline 69 if not line.strip(): continue 70 if line.strip().startswith('#'): continue 71 if entireBlock: 72 var=line.split(":")[0].strip() 73 if not var in ['test','testset','build']: 74 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 75 nspace=len(line)-len(line.lstrip(stripstr)) 76 newline=line[nspace:] 77 break 78 79 # Strip off any indentation for the whole string and any trailing 80 # whitespace for convenience 81 newTestStr="\n" 82 if len(fileNums)>0: lineNum=fileNums[0]-1 83 firstPass=True 84 for lline in block.split("\n"): 85 if len(fileNums)>0: lineNum+=1 86 line=lline[1:] if lline.startswith("!") else lline 87 if not line.strip(): continue 88 if line.strip().startswith('#'): 89 newTestStr+=line+'\n' 90 else: 91 newline=line[nspace:] 92 newTestStr+=newline.rstrip()+"\n" 93 # Do some basic indentation checks 94 if entireBlock: 95 # Don't need to check comment lines 96 if line.strip().startswith('#'): continue 97 if not newline.startswith(" "): 98 var=newline.split(":")[0].strip() 99 if not var in ['test','testset','build']: 100 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 101 if len(fileNums)>0: 102 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 103 else: 104 raise Exception(err) 105 else: 106 var=line.split(":")[0].strip() 107 if var in ['test','testset','build']: 108 subnspace=len(line)-len(line.lstrip(stripstr)) 109 if firstPass: 110 firstsubnspace=subnspace 111 firstPass=False 112 else: 113 if firstsubnspace!=subnspace: 114 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 115 if len(fileNums)>0: 116 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 117 else: 118 raise Exception(err) 119 120 # Allow line continuation character '\' 121 return newTestStr.replace('\\\n', ' ') 122 123def parseLoopArgs(varset): 124 """ 125 Given: String containing loop variables 126 Return: tuple containing separate/shared and string of loop vars 127 """ 128 keynm=varset.split("{{")[0].strip().lstrip('-') 129 if not keynm.strip(): keynm='nsize' 130 lvars=varset.split('{{')[1].split('}')[0] 131 suffx=varset.split('{{')[1].split('}')[1] 132 ftype='separate' if suffx.startswith('separate') else 'shared' 133 return keynm,lvars,ftype 134 135def _getSeparateTestvars(testDict): 136 """ 137 Given: dictionary that may have 138 Return: Variables that cause a test split 139 """ 140 vals=None 141 sepvars=[] 142 # Check nsize 143 if 'nsize' in testDict: 144 varset=testDict['nsize'] 145 if '{{' in varset: 146 keynm,lvars,ftype=parseLoopArgs(varset) 147 if ftype=='separate': sepvars.append(keynm) 148 149 # Now check args 150 if 'args' not in testDict: return sepvars 151 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 152 if not varset.strip(): continue 153 if '{{' in varset: 154 # Assuming only one for loop per var specification 155 keynm,lvars,ftype=parseLoopArgs(varset) 156 if ftype=='separate': sepvars.append(keynm) 157 158 return sepvars 159 160def _getNewArgs(args): 161 """ 162 Given: String that has args that might have loops in them 163 Return: All of the arguments/values that do not have 164 for 'separate output' in for loops 165 """ 166 newargs='' 167 if not args.strip(): return args 168 for varset in re.split('-(?=[a-zA-Z])',args): 169 if not varset.strip(): continue 170 if '{{' not in varset: 171 if 'separate' not in varset: 172 newargs+="-"+varset.strip()+" " 173 174 return newargs 175 176def _getVarVals(findvar,testDict): 177 """ 178 Given: variable that is either nsize or in args 179 Return: Values to loop over and the other arguments 180 Note that we keep the other arguments even if they have 181 for loops to enable stepping through all of the for lops 182 """ 183 save_vals=None 184 if findvar=='nsize': 185 varset=testDict[findvar] 186 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 187 else: 188 varlist=[] 189 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 190 if not varset.strip(): continue 191 if '{{' not in varset: continue 192 keyvar,vals,ftype=parseLoopArgs(varset) 193 if keyvar==findvar: 194 save_vals=vals 195 196 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 197 return save_vals 198 199def genTestsSeparateTestvars(intests,indicts,final=False): 200 """ 201 Given: testname, sdict with 'separate_testvars 202 Return: testnames,sdicts: List of generated tests 203 The tricky part here is the {{ ... }separate output} 204 that can be used multiple times 205 """ 206 testnames=[]; sdicts=[] 207 for i in range(len(intests)): 208 testname=intests[i]; sdict=indicts[i]; i+=1 209 separate_testvars=_getSeparateTestvars(sdict) 210 if len(separate_testvars)>0: 211 sep_dicts=[sdict.copy()] 212 if 'args' in sep_dicts[0]: 213 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 214 sep_testnames=[testname] 215 for kvar in separate_testvars: 216 kvals=_getVarVals(kvar,sdict) 217 218 # Have to do loop over previous var/val combos as well 219 # and accumulate as we go 220 val_testnames=[]; val_dicts=[] 221 for val in kvals.split(): 222 gensuffix="_"+kvar+"-"+val.replace(',','__') 223 for kvaltestnm in sep_testnames: 224 val_testnames.append(kvaltestnm+gensuffix) 225 for kv in sep_dicts: 226 kvardict=kv.copy() 227 # If the last var then we have the final version 228 if 'suffix' in sdict: 229 kvardict['suffix']+=gensuffix 230 else: 231 kvardict['suffix']=gensuffix 232 if kvar=='nsize': 233 kvardict[kvar]=val 234 else: 235 kvardict['args']+="-"+kvar+" "+val+" " 236 val_dicts.append(kvardict) 237 sep_testnames=val_testnames 238 sep_dicts=val_dicts 239 testnames+=sep_testnames 240 sdicts+=sep_dicts 241 else: 242 # These are plain vanilla tests (no subtests, no loops) that 243 # do not have a suffix. This makes the targets match up with 244 # the output file (testname_1.out) 245 if final: 246 if '_' not in testname: testname+='_1' 247 testnames.append(testname) 248 sdicts.append(sdict) 249 return testnames,sdicts 250 251def genTestsSubtestSuffix(testnames,sdicts): 252 """ 253 Given: testname, sdict with separate_testvars 254 Return: testnames,sdicts: List of generated tests 255 """ 256 tnms=[]; sdcts=[] 257 for i in range(len(testnames)): 258 testname=testnames[i] 259 rmsubtests=[]; keepSubtests=False 260 if 'subtests' in sdicts[i]: 261 for stest in sdicts[i]["subtests"]: 262 if 'suffix' in sdicts[i][stest]: 263 rmsubtests.append(stest) 264 gensuffix="_"+sdicts[i][stest]['suffix'] 265 newtestnm=testname+gensuffix 266 tnms.append(newtestnm) 267 newsdict=sdicts[i].copy() 268 del newsdict['subtests'] 269 # Have to hand update 270 # Append 271 for kup in appendlist: 272 if kup in sdicts[i][stest]: 273 if kup in sdicts[i]: 274 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 275 else: 276 newsdict[kup]=sdicts[i][stest][kup] 277 # Promote 278 for kup in acceptedkeys: 279 if kup in appendlist: continue 280 if kup in sdicts[i][stest]: 281 newsdict[kup]=sdicts[i][stest][kup] 282 # Cleanup 283 for st in sdicts[i]["subtests"]: del newsdict[st] 284 sdcts.append(newsdict) 285 else: 286 keepSubtests=True 287 else: 288 tnms.append(testnames[i]) 289 sdcts.append(sdicts[i]) 290 # If a subtest without a suffix exists, then save it 291 if keepSubtests: 292 tnms.append(testnames[i]) 293 newsdict=sdicts[i].copy() 294 # Prune the tests to prepare for keeping 295 for rmtest in rmsubtests: 296 newsdict['subtests'].remove(rmtest) 297 del newsdict[rmtest] 298 sdcts.append(newsdict) 299 i+=1 300 return tnms,sdcts 301 302def splitTests(testname,sdict): 303 """ 304 Given: testname and YAML-generated dictionary 305 Return: list of names and dictionaries corresponding to each test 306 given that the YAML language allows for multiple tests 307 """ 308 309 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 310 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 311 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 312 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 313 314 # Because I am altering the list, I do this in passes. Inelegant 315 316 return testnames, sdicts 317 318def parseTest(testStr,srcfile,verbosity): 319 """ 320 This parses an individual test 321 YAML is hierarchial so should use a state machine in the general case, 322 but in practice we only support two levels of test: 323 """ 324 basename=os.path.basename(srcfile) 325 # Handle the new at the begininng 326 bn=re.sub("new_","",basename) 327 # This is the default 328 testname="run"+os.path.splitext(bn)[0] 329 330 # Tests that have default everything (so empty effectively) 331 if len(testStr)==0: return [testname], [{}] 332 333 striptest=_stripIndent(testStr,srcfile) 334 335 # go through and parse 336 subtestnum=0 337 subdict={} 338 comments=[] 339 indentlevel=0 340 for ln in striptest.split("\n"): 341 line=ln.split('#')[0].rstrip() 342 if verbosity>2: print(line) 343 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 344 if comment: comments.append(comment) 345 if not line.strip(): continue 346 lsplit=line.split(':') 347 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 348 indentcount=lsplit[0].count(" ") 349 var=lsplit[0].strip() 350 val=line[line.find(':')+1:].strip() 351 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 352 # Start by seeing if we are in a subtest 353 if line.startswith(" "): 354 if var in subdict[subtestname]: 355 subdict[subtestname][var]+=" "+val 356 else: 357 subdict[subtestname][var]=val 358 if not indentlevel: indentlevel=indentcount 359 #if indentlevel!=indentcount: print("Error in indentation:", ln) 360 # Determine subtest name and make dict 361 elif var=="test": 362 subtestname="test"+str(subtestnum) 363 subdict[subtestname]={} 364 if "subtests" not in subdict: subdict["subtests"]=[] 365 subdict["subtests"].append(subtestname) 366 subtestnum=subtestnum+1 367 # The rest are easy 368 else: 369 # For convenience, it is sometimes convenient to list twice 370 if var in subdict: 371 if var in appendlist: 372 subdict[var]+=" "+val 373 else: 374 raise Exception(var+" entered twice: "+line) 375 else: 376 subdict[var]=val 377 if var=="suffix": 378 if len(val)>0: 379 testname+="_"+val 380 381 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 382 #if "_" not in testname: testname+='_1' 383 384 # A test block can create multiple tests. This does that logic 385 testnames,subdicts=splitTests(testname,subdict) 386 return testnames,subdicts 387 388def parseTests(testStr,srcfile,fileNums,verbosity): 389 """ 390 Parse the yaml string describing tests and return 391 a dictionary with the info in the form of: 392 testDict[test][subtest] 393 This is an inelegant parser as we do not wish to 394 introduce a new dependency by bringing in pyyaml. 395 The advantage is that validation can be done as 396 it is parsed (e.g., 'test' is the only top-level node) 397 """ 398 399 testDict={} 400 401 # The first entry should be test: but it might be indented. 402 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 403 if verbosity>2: print(srcfile) 404 405 ## Check and see if we have build requirements 406 addToRunRequirements=None 407 if "\nbuild:" in newTestStr: 408 testDict['build']={} 409 # The file info is already here and need to append 410 Part1=newTestStr.split("build:")[1] 411 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 412 for bkey in buildkeys: 413 if bkey+":" in fileInfo: 414 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 415 #if verbosity>1: bkey+": "+testDict['build'][bkey] 416 # If a runtime requires are put into build, push them down to all run tests 417 # At this point, we are working with strings and not lists 418 if 'requires' in testDict['build']: 419 if 'datafilespath' in testDict['build']['requires']: 420 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 421 testDict['build']['requires']=newreqs.strip() 422 addToRunRequirements='datafilespath' 423 424 425 # Now go through each test. First elem in split is blank 426 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 427 testnames,subdicts=parseTest(test,srcfile,verbosity) 428 for i in range(len(testnames)): 429 if testnames[i] in testDict: 430 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 431 # Add in build requirements that need to be moved 432 if addToRunRequirements: 433 if 'requires' in subdicts[i]: 434 subdicts[i]['requires']+=addToRunRequirements 435 else: 436 subdicts[i]['requires']=addToRunRequirements 437 testDict[testnames[i]]=subdicts[i] 438 439 return testDict 440 441def parseTestFile(srcfile,verbosity): 442 """ 443 Parse single example files and return dictionary of the form: 444 testDict[srcfile][test][subtest] 445 """ 446 debug=False 447 basename=os.path.basename(srcfile) 448 if basename=='makefile': return {} 449 450 curdir=os.path.realpath(os.path.curdir) 451 basedir=os.path.dirname(os.path.realpath(srcfile)) 452 os.chdir(basedir) 453 454 testDict={} 455 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 456 457 ## Start with doing the tests 458 # 459 fsplit=fileStr.split("/*TEST\n")[1:] 460 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 461 # Allow for multiple "/*TEST" blocks even though it really should be 462 # one 463 srcTests=[] 464 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 465 testString=" ".join(srcTests) 466 flen=len(testString.split("\n")) 467 fend=fstart+flen-1 468 fileNums=range(fstart,fend) 469 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 470 # Massage dictionary for build requirements 471 if 'build' in testDict[basename]: 472 testDict[basename].update(testDict[basename]['build']) 473 del testDict[basename]['build'] 474 475 476 os.chdir(curdir) 477 return testDict 478 479def parseTestDir(directory,verbosity): 480 """ 481 Parse single example files and return dictionary of the form: 482 testDict[srcfile][test][subtest] 483 """ 484 curdir=os.path.realpath(os.path.curdir) 485 basedir=os.path.realpath(directory) 486 os.chdir(basedir) 487 488 tDict={} 489 for test_file in sorted(glob.glob("new_ex*.*")): 490 tDict.update(parseTestFile(test_file,verbosity)) 491 492 os.chdir(curdir) 493 return tDict 494 495def printExParseDict(rDict): 496 """ 497 This is useful for debugging 498 """ 499 indent=" " 500 for sfile in rDict: 501 print(sfile) 502 sortkeys=list(rDict[sfile].keys()) 503 sortkeys.sort() 504 for runex in sortkeys: 505 print(indent+runex) 506 if type(rDict[sfile][runex])==bytes: 507 print(indent*2+rDict[sfile][runex]) 508 else: 509 for var in rDict[sfile][runex]: 510 if var.startswith("test"): continue 511 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 512 if 'subtests' in rDict[sfile][runex]: 513 for var in rDict[sfile][runex]['subtests']: 514 print(indent*2+var) 515 for var2 in rDict[sfile][runex][var]: 516 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 517 print("\n") 518 return 519 520def main(directory='',test_file='',verbosity=0): 521 522 if directory: 523 tDict=parseTestDir(directory,verbosity) 524 else: 525 tDict=parseTestFile(test_file,verbosity) 526 if verbosity>0: printExParseDict(tDict) 527 528 return 529 530if __name__ == '__main__': 531 import optparse 532 parser = optparse.OptionParser() 533 parser.add_option('-d', '--directory', dest='directory', 534 default="", help='Directory containing files to parse') 535 parser.add_option('-t', '--test_file', dest='test_file', 536 default="", help='Test file, e.g., ex1.c, to parse') 537 parser.add_option('-v', '--verbosity', dest='verbosity', 538 help='Verbosity of output by level: 1, 2, or 3', default=0) 539 opts, extra_args = parser.parse_args() 540 541 if extra_args: 542 import sys 543 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 544 exit(1) 545 if not opts.test_file and not opts.directory: 546 print("test file or directory is required") 547 parser.print_usage() 548 sys.exit() 549 550 # Need verbosity to be an integer 551 try: 552 verbosity=int(opts.verbosity) 553 except: 554 raise Exception("Error: Verbosity must be integer") 555 556 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 557