1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args requires comments".split() 54 55import re 56 57def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 58 """ 59 Go through and remove a level of indentation 60 Also strip of trailing whitespace 61 """ 62 # The first entry should be test: but it might be indented. 63 ext=os.path.splitext(srcfile)[1] 64 stripstr=" " 65 if len(fileNums)>0: lineNum=fileNums[0]-1 66 for lline in block.split("\n"): 67 if len(fileNums)>0: lineNum+=1 68 line=lline[1:] if lline.startswith("!") else lline 69 if not line.strip(): continue 70 if line.strip().startswith('#'): continue 71 if entireBlock: 72 var=line.split(":")[0].strip() 73 if not var in ['test','testset','build']: 74 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 75 nspace=len(line)-len(line.lstrip(stripstr)) 76 newline=line[nspace:] 77 break 78 79 # Strip off any indentation for the whole string and any trailing 80 # whitespace for convenience 81 newTestStr="\n" 82 if len(fileNums)>0: lineNum=fileNums[0]-1 83 firstPass=True 84 for lline in block.split("\n"): 85 if len(fileNums)>0: lineNum+=1 86 line=lline[1:] if lline.startswith("!") else lline 87 if not line.strip(): continue 88 if line.strip().startswith('#'): 89 newTestStr+=line+'\n' 90 else: 91 newline=line[nspace:] 92 newTestStr+=newline.rstrip()+"\n" 93 # Do some basic indentation checks 94 if entireBlock: 95 # Don't need to check comment lines 96 if line.strip().startswith('#'): continue 97 if not newline.startswith(" "): 98 var=newline.split(":")[0].strip() 99 if not var in ['test','testset','build']: 100 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 101 if len(fileNums)>0: 102 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 103 else: 104 raise Exception(err) 105 else: 106 var=line.split(":")[0].strip() 107 if var in ['test','testset','build']: 108 subnspace=len(line)-len(line.lstrip(stripstr)) 109 if firstPass: 110 firstsubnspace=subnspace 111 firstPass=False 112 else: 113 if firstsubnspace!=subnspace: 114 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 115 if len(fileNums)>0: 116 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 117 else: 118 raise Exception(err) 119 120 # Allow line continuation character '\' 121 return newTestStr.replace('\\\n', ' ') 122 123def parseLoopArgs(varset): 124 """ 125 Given: String containing loop variables 126 Return: tuple containing separate/shared and string of loop vars 127 """ 128 keynm=varset.split("{{")[0].strip() 129 if not keynm.strip(): keynm='nsize' 130 lvars=varset.split('{{')[1].split('}')[0] 131 suffx=varset.split('{{')[1].split('}')[1] 132 ftype='separate' if suffx.startswith('separate') else 'shared' 133 return keynm,lvars,ftype 134 135def _getSeparateTestvars(testDict): 136 """ 137 Given: dictionary that may have 138 Return: Variables that cause a test split 139 """ 140 vals=None 141 sepvars=[] 142 # Check nsize 143 if 'nsize' in testDict: 144 varset=testDict['nsize'] 145 if '{{' in varset: 146 keynm,lvars,ftype=parseLoopArgs(varset) 147 if ftype=='separate': sepvars.append(keynm) 148 149 # Now check args 150 if 'args' not in testDict: return sepvars 151 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 152 if not varset.strip(): continue 153 if '{{' in varset: 154 # Assuming only one for loop per var specification 155 keynm,lvars,ftype=parseLoopArgs(varset) 156 if ftype=='separate': sepvars.append(keynm) 157 158 return sepvars 159 160def _getNewArgs(args): 161 """ 162 Given: String that has args that might have loops in them 163 Return: All of the arguments/values that do not have 164 for 'separate output' in for loops 165 """ 166 newargs='' 167 if not args.strip(): return args 168 for varset in re.split('-(?=[a-zA-Z])',args): 169 if not varset.strip(): continue 170 if '{{' not in varset: 171 if 'separate' not in varset: 172 newargs+="-"+varset.strip()+" " 173 174 return newargs 175 176def _getVarVals(findvar,testDict): 177 """ 178 Given: variable that is either nsize or in args 179 Return: Values to loop over and the other arguments 180 Note that we keep the other arguments even if they have 181 for loops to enable stepping through all of the for lops 182 """ 183 save_vals=None 184 if findvar=='nsize': 185 varset=testDict[findvar] 186 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 187 else: 188 varlist=[] 189 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 190 if not varset.strip(): continue 191 if '{{' not in varset: continue 192 keyvar,vals,ftype=parseLoopArgs(varset) 193 if keyvar==findvar: 194 save_vals=vals 195 196 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 197 return save_vals 198 199def genTestsSeparateTestvars(intests,indicts): 200 """ 201 Given: testname, sdict with 'separate_testvars 202 Return: testnames,sdicts: List of generated tests 203 The tricky part here is the {{ ... }separate output} 204 that can be used multiple times 205 """ 206 testnames=[]; sdicts=[] 207 for i in range(len(intests)): 208 testname=intests[i]; sdict=indicts[i]; i+=1 209 separate_testvars=_getSeparateTestvars(sdict) 210 if len(separate_testvars)>0: 211 sep_dicts=[sdict.copy()] 212 if 'args' in sep_dicts[0]: 213 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 214 sep_testnames=[testname] 215 for kvar in separate_testvars: 216 kvals=_getVarVals(kvar,sdict) 217 218 # Have to do loop over previous var/val combos as well 219 # and accumulate as we go 220 val_testnames=[]; val_dicts=[] 221 for val in kvals.split(): 222 gensuffix="_"+kvar+"-"+val.replace(',','__') 223 for kvaltestnm in sep_testnames: 224 val_testnames.append(kvaltestnm+gensuffix) 225 for kv in sep_dicts: 226 kvardict=kv.copy() 227 # If the last var then we have the final version 228 if 'suffix' in sdict: 229 kvardict['suffix']+=gensuffix 230 else: 231 kvardict['suffix']=gensuffix 232 if kvar=='nsize': 233 kvardict[kvar]=val 234 else: 235 kvardict['args']+="-"+kvar+" "+val+" " 236 val_dicts.append(kvardict) 237 sep_testnames=val_testnames 238 sep_dicts=val_dicts 239 testnames+=sep_testnames 240 sdicts+=sep_dicts 241 else: 242 testnames.append(testname) 243 sdicts.append(sdict) 244 return testnames,sdicts 245 246def genTestsSubtestSuffix(testnames,sdicts): 247 """ 248 Given: testname, sdict with separate_testvars 249 Return: testnames,sdicts: List of generated tests 250 """ 251 tnms=[]; sdcts=[] 252 for i in range(len(testnames)): 253 testname=testnames[i] 254 rmsubtests=[]; keepSubtests=False 255 if 'subtests' in sdicts[i]: 256 for stest in sdicts[i]["subtests"]: 257 if 'suffix' in sdicts[i][stest]: 258 rmsubtests.append(stest) 259 gensuffix="_"+sdicts[i][stest]['suffix'] 260 newtestnm=testname+gensuffix 261 tnms.append(newtestnm) 262 newsdict=sdicts[i].copy() 263 del newsdict['subtests'] 264 # Have to hand update 265 # Append 266 for kup in appendlist: 267 if kup in sdicts[i][stest]: 268 if kup in sdicts[i]: 269 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 270 else: 271 newsdict[kup]=sdicts[i][stest][kup] 272 # Promote 273 for kup in acceptedkeys: 274 if kup in appendlist: continue 275 if kup in sdicts[i][stest]: 276 newsdict[kup]=sdicts[i][stest][kup] 277 # Cleanup 278 for st in sdicts[i]["subtests"]: del newsdict[st] 279 sdcts.append(newsdict) 280 else: 281 keepSubtests=True 282 else: 283 tnms.append(testnames[i]) 284 sdcts.append(sdicts[i]) 285 # If a subtest without a suffix exists, then save it 286 if keepSubtests: 287 tnms.append(testnames[i]) 288 newsdict=sdicts[i].copy() 289 # Prune the tests to prepare for keeping 290 for rmtest in rmsubtests: 291 newsdict['subtests'].remove(rmtest) 292 del newsdict[rmtest] 293 sdcts.append(newsdict) 294 i+=1 295 return tnms,sdcts 296 297def splitTests(testname,sdict): 298 """ 299 Given: testname and YAML-generated dictionary 300 Return: list of names and dictionaries corresponding to each test 301 given that the YAML language allows for multiple tests 302 """ 303 304 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 305 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 306 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 307 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts) 308 309 # Because I am altering the list, I do this in passes. Inelegant 310 311 return testnames, sdicts 312 313def parseTest(testStr,srcfile,verbosity): 314 """ 315 This parses an individual test 316 YAML is hierarchial so should use a state machine in the general case, 317 but in practice we only support two levels of test: 318 """ 319 basename=os.path.basename(srcfile) 320 # Handle the new at the begininng 321 bn=re.sub("new_","",basename) 322 # This is the default 323 testname="run"+os.path.splitext(bn)[0] 324 325 # Tests that have default everything (so empty effectively) 326 if len(testStr)==0: return [testname], [{}] 327 328 striptest=_stripIndent(testStr,srcfile) 329 330 # go through and parse 331 subtestnum=0 332 subdict={} 333 comments=[] 334 indentlevel=0 335 for ln in striptest.split("\n"): 336 line=ln.split('#')[0].rstrip() 337 if verbosity>2: print(line) 338 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 339 if comment: comments.append(comment) 340 if not line.strip(): continue 341 lsplit=line.split(':') 342 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 343 indentcount=lsplit[0].count(" ") 344 var=lsplit[0].strip() 345 val=line[line.find(':')+1:].strip() 346 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 347 # Start by seeing if we are in a subtest 348 if line.startswith(" "): 349 subdict[subtestname][var]=val 350 if not indentlevel: indentlevel=indentcount 351 #if indentlevel!=indentcount: print("Error in indentation:", ln) 352 # Determine subtest name and make dict 353 elif var=="test": 354 subtestname="test"+str(subtestnum) 355 subdict[subtestname]={} 356 if "subtests" not in subdict: subdict["subtests"]=[] 357 subdict["subtests"].append(subtestname) 358 subtestnum=subtestnum+1 359 # The rest are easy 360 else: 361 # For convenience, it is sometimes convenient to list twice 362 if var in subdict: 363 if var in appendlist: 364 subdict[var]+=" "+val 365 else: 366 raise Exception(var+" entered twice: "+line) 367 else: 368 subdict[var]=val 369 if var=="suffix": 370 if len(val)>0: 371 testname=testname+"_"+val 372 373 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 374 # A test block can create multiple tests. This does 375 # that logic 376 testnames,subdicts=splitTests(testname,subdict) 377 return testnames,subdicts 378 379def parseTests(testStr,srcfile,fileNums,verbosity): 380 """ 381 Parse the yaml string describing tests and return 382 a dictionary with the info in the form of: 383 testDict[test][subtest] 384 This is an inelegant parser as we do not wish to 385 introduce a new dependency by bringing in pyyaml. 386 The advantage is that validation can be done as 387 it is parsed (e.g., 'test' is the only top-level node) 388 """ 389 390 testDict={} 391 392 # The first entry should be test: but it might be indented. 393 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 394 if verbosity>2: print(srcfile) 395 396 ## Check and see if we have build requirements 397 addToRunRequirements=None 398 if "\nbuild:" in newTestStr: 399 testDict['build']={} 400 # The file info is already here and need to append 401 Part1=newTestStr.split("build:")[1] 402 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 403 for bkey in buildkeys: 404 if bkey+":" in fileInfo: 405 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 406 #if verbosity>1: bkey+": "+testDict['build'][bkey] 407 # If a runtime requires are put into build, push them down to all run tests 408 # At this point, we are working with strings and not lists 409 if 'requires' in testDict['build']: 410 if 'datafilespath' in testDict['build']['requires']: 411 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 412 testDict['build']['requires']=newreqs.strip() 413 addToRunRequirements='datafilespath' 414 415 416 # Now go through each test. First elem in split is blank 417 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 418 testnames,subdicts=parseTest(test,srcfile,verbosity) 419 for i in range(len(testnames)): 420 if testnames[i] in testDict: 421 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 422 # Add in build requirements that need to be moved 423 if addToRunRequirements: 424 if 'requires' in subdicts[i]: 425 subdicts[i]['requires']+=addToRunRequirements 426 else: 427 subdicts[i]['requires']=addToRunRequirements 428 testDict[testnames[i]]=subdicts[i] 429 430 return testDict 431 432def parseTestFile(srcfile,verbosity): 433 """ 434 Parse single example files and return dictionary of the form: 435 testDict[srcfile][test][subtest] 436 """ 437 debug=False 438 basename=os.path.basename(srcfile) 439 if basename=='makefile': return {} 440 441 curdir=os.path.realpath(os.path.curdir) 442 basedir=os.path.dirname(os.path.realpath(srcfile)) 443 os.chdir(basedir) 444 445 testDict={} 446 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 447 448 ## Start with doing the tests 449 # 450 fsplit=fileStr.split("/*TEST\n")[1:] 451 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 452 # Allow for multiple "/*TEST" blocks even though it really should be 453 # one 454 srcTests=[] 455 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 456 testString=" ".join(srcTests) 457 flen=len(testString.split("\n")) 458 fend=fstart+flen-1 459 fileNums=range(fstart,fend) 460 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 461 # Massage dictionary for build requirements 462 if 'build' in testDict[basename]: 463 testDict[basename].update(testDict[basename]['build']) 464 del testDict[basename]['build'] 465 466 467 os.chdir(curdir) 468 return testDict 469 470def parseTestDir(directory,verbosity): 471 """ 472 Parse single example files and return dictionary of the form: 473 testDict[srcfile][test][subtest] 474 """ 475 curdir=os.path.realpath(os.path.curdir) 476 basedir=os.path.realpath(directory) 477 os.chdir(basedir) 478 479 tDict={} 480 for test_file in sorted(glob.glob("new_ex*.*")): 481 tDict.update(parseTestFile(test_file,verbosity)) 482 483 os.chdir(curdir) 484 return tDict 485 486def printExParseDict(rDict): 487 """ 488 This is useful for debugging 489 """ 490 indent=" " 491 for sfile in rDict: 492 print(sfile) 493 sortkeys=rDict[sfile].keys() 494 sortkeys.sort() 495 for runex in sortkeys: 496 print(indent+runex) 497 if type(rDict[sfile][runex])==bytes: 498 print(indent*2+rDict[sfile][runex]) 499 else: 500 for var in rDict[sfile][runex]: 501 if var.startswith("test"): continue 502 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 503 if 'subtests' in rDict[sfile][runex]: 504 for var in rDict[sfile][runex]['subtests']: 505 print(indent*2+var) 506 for var2 in rDict[sfile][runex][var]: 507 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 508 print("\n") 509 return 510 511def main(directory='',test_file='',verbosity=0): 512 513 if directory: 514 tDict=parseTestDir(directory,verbosity) 515 else: 516 tDict=parseTestFile(test_file,verbosity) 517 if verbosity>0: printExParseDict(tDict) 518 519 return 520 521if __name__ == '__main__': 522 import optparse 523 parser = optparse.OptionParser() 524 parser.add_option('-d', '--directory', dest='directory', 525 default="", help='Directory containing files to parse') 526 parser.add_option('-t', '--test_file', dest='test_file', 527 default="", help='Test file, e.g., ex1.c, to parse') 528 parser.add_option('-v', '--verbosity', dest='verbosity', 529 help='Verbosity of output by level: 1, 2, or 3', default=0) 530 opts, extra_args = parser.parse_args() 531 532 if extra_args: 533 import sys 534 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 535 exit(1) 536 if not opts.test_file and not opts.directory: 537 print("test file or directory is required") 538 parser.print_usage() 539 sys.exit() 540 541 # Need verbosity to be an integer 542 try: 543 verbosity=int(opts.verbosity) 544 except: 545 raise Exception("Error: Verbosity must be integer") 546 547 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 548