1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args requires comments".split() 54 55import re 56 57def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 58 """ 59 Go through and remove a level of indentation 60 Also strip of trailing whitespace 61 """ 62 # The first entry should be test: but it might be indented. 63 ext=os.path.splitext(srcfile)[1] 64 stripstr=" " 65 if len(fileNums)>0: lineNum=fileNums[0]-1 66 for lline in block.split("\n"): 67 if len(fileNums)>0: lineNum+=1 68 line=lline[1:] if lline.startswith("!") else lline 69 if not line.strip(): continue 70 if line.strip().startswith('#'): continue 71 if entireBlock: 72 var=line.split(":")[0].strip() 73 if not var in ['test','testset','build']: 74 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 75 nspace=len(line)-len(line.lstrip(stripstr)) 76 newline=line[nspace:] 77 break 78 79 # Strip off any indentation for the whole string and any trailing 80 # whitespace for convenience 81 newTestStr="\n" 82 if len(fileNums)>0: lineNum=fileNums[0]-1 83 firstPass=True 84 for lline in block.split("\n"): 85 if len(fileNums)>0: lineNum+=1 86 line=lline[1:] if lline.startswith("!") else lline 87 if not line.strip(): continue 88 if line.strip().startswith('#'): 89 newTestStr+=line+'\n' 90 else: 91 newline=line[nspace:] 92 newTestStr+=newline.rstrip()+"\n" 93 # Do some basic indentation checks 94 if entireBlock: 95 # Don't need to check comment lines 96 if line.strip().startswith('#'): continue 97 if not newline.startswith(" "): 98 var=newline.split(":")[0].strip() 99 if not var in ['test','testset','build']: 100 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 101 if len(fileNums)>0: 102 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 103 else: 104 raise Exception(err) 105 else: 106 var=line.split(":")[0].strip() 107 if var in ['test','testset','build']: 108 subnspace=len(line)-len(line.lstrip(stripstr)) 109 if firstPass: 110 firstsubnspace=subnspace 111 firstPass=False 112 else: 113 if firstsubnspace!=subnspace: 114 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 115 if len(fileNums)>0: 116 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 117 else: 118 raise Exception(err) 119 120 # Allow line continuation character '\' 121 return newTestStr.replace('\\\n', ' ') 122 123def parseLoopArgs(varset): 124 """ 125 Given: String containing loop variables 126 Return: tuple containing separate/shared and string of loop vars 127 """ 128 keynm=varset.split("{{")[0].strip().lstrip('-') 129 if not keynm.strip(): keynm='nsize' 130 lvars=varset.split('{{')[1].split('}')[0] 131 suffx=varset.split('{{')[1].split('}')[1] 132 ftype='separate' if suffx.startswith('separate') else 'shared' 133 return keynm,lvars,ftype 134 135def _getSeparateTestvars(testDict): 136 """ 137 Given: dictionary that may have 138 Return: Variables that cause a test split 139 """ 140 vals=None 141 sepvars=[] 142 # Check nsize 143 if 'nsize' in testDict: 144 varset=testDict['nsize'] 145 if '{{' in varset: 146 keynm,lvars,ftype=parseLoopArgs(varset) 147 if ftype=='separate': sepvars.append(keynm) 148 149 # Now check args 150 if 'args' not in testDict: return sepvars 151 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 152 if not varset.strip(): continue 153 if '{{' in varset: 154 # Assuming only one for loop per var specification 155 keynm,lvars,ftype=parseLoopArgs(varset) 156 if ftype=='separate': sepvars.append(keynm) 157 158 return sepvars 159 160def _getNewArgs(args): 161 """ 162 Given: String that has args that might have loops in them 163 Return: All of the arguments/values that do not have 164 for 'separate output' in for loops 165 """ 166 newargs='' 167 if not args.strip(): return args 168 for varset in re.split('-(?=[a-zA-Z])',args): 169 if not varset.strip(): continue 170 if '{{' not in varset: 171 if 'separate' not in varset: 172 newargs+="-"+varset.strip()+" " 173 174 return newargs 175 176def _getVarVals(findvar,testDict): 177 """ 178 Given: variable that is either nsize or in args 179 Return: Values to loop over and the other arguments 180 Note that we keep the other arguments even if they have 181 for loops to enable stepping through all of the for lops 182 """ 183 save_vals=None 184 if findvar=='nsize': 185 varset=testDict[findvar] 186 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 187 else: 188 varlist=[] 189 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 190 if not varset.strip(): continue 191 if '{{' not in varset: continue 192 keyvar,vals,ftype=parseLoopArgs(varset) 193 if keyvar==findvar: 194 save_vals=vals 195 196 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 197 return save_vals 198 199def genTestsSeparateTestvars(intests,indicts,final=False): 200 """ 201 Given: testname, sdict with 'separate_testvars 202 Return: testnames,sdicts: List of generated tests 203 The tricky part here is the {{ ... }separate output} 204 that can be used multiple times 205 """ 206 testnames=[]; sdicts=[] 207 for i in range(len(intests)): 208 testname=intests[i]; sdict=indicts[i]; i+=1 209 separate_testvars=_getSeparateTestvars(sdict) 210 if len(separate_testvars)>0: 211 sep_dicts=[sdict.copy()] 212 if 'args' in sep_dicts[0]: 213 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 214 sep_testnames=[testname] 215 for kvar in separate_testvars: 216 kvals=_getVarVals(kvar,sdict) 217 218 # Have to do loop over previous var/val combos as well 219 # and accumulate as we go 220 val_testnames=[]; val_dicts=[] 221 for val in kvals.split(): 222 gensuffix="_"+kvar+"-"+val.replace(',','__') 223 for kvaltestnm in sep_testnames: 224 val_testnames.append(kvaltestnm+gensuffix) 225 for kv in sep_dicts: 226 kvardict=kv.copy() 227 # If the last var then we have the final version 228 if 'suffix' in sdict: 229 kvardict['suffix']+=gensuffix 230 else: 231 kvardict['suffix']=gensuffix 232 if kvar=='nsize': 233 kvardict[kvar]=val 234 else: 235 kvardict['args']+="-"+kvar+" "+val+" " 236 val_dicts.append(kvardict) 237 sep_testnames=val_testnames 238 sep_dicts=val_dicts 239 testnames+=sep_testnames 240 sdicts+=sep_dicts 241 else: 242 # These are plain vanilla tests (no subtests, no loops) that 243 # do not have a suffix. This makes the targets match up with 244 # the output file (testname_1.out) 245 if final: 246 if '_' not in testname: testname+='_1' 247 testnames.append(testname) 248 sdicts.append(sdict) 249 return testnames,sdicts 250 251def genTestsSubtestSuffix(testnames,sdicts): 252 """ 253 Given: testname, sdict with separate_testvars 254 Return: testnames,sdicts: List of generated tests 255 """ 256 tnms=[]; sdcts=[] 257 for i in range(len(testnames)): 258 testname=testnames[i] 259 rmsubtests=[]; keepSubtests=False 260 if 'subtests' in sdicts[i]: 261 for stest in sdicts[i]["subtests"]: 262 if 'suffix' in sdicts[i][stest]: 263 rmsubtests.append(stest) 264 gensuffix="_"+sdicts[i][stest]['suffix'] 265 newtestnm=testname+gensuffix 266 tnms.append(newtestnm) 267 newsdict=sdicts[i].copy() 268 del newsdict['subtests'] 269 # Have to hand update 270 # Append 271 for kup in appendlist: 272 if kup in sdicts[i][stest]: 273 if kup in sdicts[i]: 274 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 275 else: 276 newsdict[kup]=sdicts[i][stest][kup] 277 # Promote 278 for kup in acceptedkeys: 279 if kup in appendlist: continue 280 if kup in sdicts[i][stest]: 281 newsdict[kup]=sdicts[i][stest][kup] 282 # Cleanup 283 for st in sdicts[i]["subtests"]: del newsdict[st] 284 sdcts.append(newsdict) 285 else: 286 keepSubtests=True 287 else: 288 tnms.append(testnames[i]) 289 sdcts.append(sdicts[i]) 290 # If a subtest without a suffix exists, then save it 291 if keepSubtests: 292 tnms.append(testnames[i]) 293 newsdict=sdicts[i].copy() 294 # Prune the tests to prepare for keeping 295 for rmtest in rmsubtests: 296 newsdict['subtests'].remove(rmtest) 297 del newsdict[rmtest] 298 sdcts.append(newsdict) 299 i+=1 300 return tnms,sdcts 301 302def splitTests(testname,sdict): 303 """ 304 Given: testname and YAML-generated dictionary 305 Return: list of names and dictionaries corresponding to each test 306 given that the YAML language allows for multiple tests 307 """ 308 309 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 310 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 311 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 312 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 313 314 # Because I am altering the list, I do this in passes. Inelegant 315 316 return testnames, sdicts 317 318def parseTest(testStr,srcfile,verbosity): 319 """ 320 This parses an individual test 321 YAML is hierarchial so should use a state machine in the general case, 322 but in practice we only support two levels of test: 323 """ 324 basename=os.path.basename(srcfile) 325 # Handle the new at the begininng 326 bn=re.sub("new_","",basename) 327 # This is the default 328 testname="run"+os.path.splitext(bn)[0] 329 330 # Tests that have default everything (so empty effectively) 331 if len(testStr)==0: return [testname], [{}] 332 333 striptest=_stripIndent(testStr,srcfile) 334 335 # go through and parse 336 subtestnum=0 337 subdict={} 338 comments=[] 339 indentlevel=0 340 for ln in striptest.split("\n"): 341 line=ln.split('#')[0].rstrip() 342 if verbosity>2: print(line) 343 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 344 if comment: comments.append(comment) 345 if not line.strip(): continue 346 lsplit=line.split(':') 347 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 348 indentcount=lsplit[0].count(" ") 349 var=lsplit[0].strip() 350 val=line[line.find(':')+1:].strip() 351 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 352 # Start by seeing if we are in a subtest 353 if line.startswith(" "): 354 if var in subdict[subtestname]: 355 subdict[subtestname][var]+=" "+val 356 else: 357 subdict[subtestname][var]=val 358 if not indentlevel: indentlevel=indentcount 359 #if indentlevel!=indentcount: print("Error in indentation:", ln) 360 # Determine subtest name and make dict 361 elif var=="test": 362 subtestname="test"+str(subtestnum) 363 subdict[subtestname]={} 364 if "subtests" not in subdict: subdict["subtests"]=[] 365 subdict["subtests"].append(subtestname) 366 subtestnum=subtestnum+1 367 # The rest are easy 368 else: 369 # For convenience, it is sometimes convenient to list twice 370 if var in subdict: 371 if var in appendlist: 372 subdict[var]+=" "+val 373 else: 374 raise Exception(var+" entered twice: "+line) 375 else: 376 subdict[var]=val 377 if var=="suffix": 378 if len(val)>0: 379 testname+="_"+val 380 381 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 382 383 # A test block can create multiple tests. This does that logic 384 testnames,subdicts=splitTests(testname,subdict) 385 return testnames,subdicts 386 387def parseTests(testStr,srcfile,fileNums,verbosity): 388 """ 389 Parse the yaml string describing tests and return 390 a dictionary with the info in the form of: 391 testDict[test][subtest] 392 This is an inelegant parser as we do not wish to 393 introduce a new dependency by bringing in pyyaml. 394 The advantage is that validation can be done as 395 it is parsed (e.g., 'test' is the only top-level node) 396 """ 397 398 testDict={} 399 400 # The first entry should be test: but it might be indented. 401 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 402 if verbosity>2: print(srcfile) 403 404 ## Check and see if we have build requirements 405 addToRunRequirements=None 406 if "\nbuild:" in newTestStr: 407 testDict['build']={} 408 # The file info is already here and need to append 409 Part1=newTestStr.split("build:")[1] 410 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 411 for bkey in buildkeys: 412 if bkey+":" in fileInfo: 413 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 414 #if verbosity>1: bkey+": "+testDict['build'][bkey] 415 # If a runtime requires are put into build, push them down to all run tests 416 # At this point, we are working with strings and not lists 417 if 'requires' in testDict['build']: 418 if 'datafilespath' in testDict['build']['requires']: 419 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 420 testDict['build']['requires']=newreqs.strip() 421 addToRunRequirements='datafilespath' 422 423 424 # Now go through each test. First elem in split is blank 425 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 426 testnames,subdicts=parseTest(test,srcfile,verbosity) 427 for i in range(len(testnames)): 428 if testnames[i] in testDict: 429 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 430 # Add in build requirements that need to be moved 431 if addToRunRequirements: 432 if 'requires' in subdicts[i]: 433 subdicts[i]['requires']+=addToRunRequirements 434 else: 435 subdicts[i]['requires']=addToRunRequirements 436 testDict[testnames[i]]=subdicts[i] 437 438 return testDict 439 440def parseTestFile(srcfile,verbosity): 441 """ 442 Parse single example files and return dictionary of the form: 443 testDict[srcfile][test][subtest] 444 """ 445 debug=False 446 basename=os.path.basename(srcfile) 447 if basename=='makefile': return {} 448 449 curdir=os.path.realpath(os.path.curdir) 450 basedir=os.path.dirname(os.path.realpath(srcfile)) 451 os.chdir(basedir) 452 453 testDict={} 454 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 455 456 ## Start with doing the tests 457 # 458 fsplit=fileStr.split("/*TEST\n")[1:] 459 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 460 # Allow for multiple "/*TEST" blocks even though it really should be 461 # one 462 srcTests=[] 463 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 464 testString=" ".join(srcTests) 465 flen=len(testString.split("\n")) 466 fend=fstart+flen-1 467 fileNums=range(fstart,fend) 468 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 469 # Massage dictionary for build requirements 470 if 'build' in testDict[basename]: 471 testDict[basename].update(testDict[basename]['build']) 472 del testDict[basename]['build'] 473 474 475 os.chdir(curdir) 476 return testDict 477 478def parseTestDir(directory,verbosity): 479 """ 480 Parse single example files and return dictionary of the form: 481 testDict[srcfile][test][subtest] 482 """ 483 curdir=os.path.realpath(os.path.curdir) 484 basedir=os.path.realpath(directory) 485 os.chdir(basedir) 486 487 tDict={} 488 for test_file in sorted(glob.glob("new_ex*.*")): 489 tDict.update(parseTestFile(test_file,verbosity)) 490 491 os.chdir(curdir) 492 return tDict 493 494def printExParseDict(rDict): 495 """ 496 This is useful for debugging 497 """ 498 indent=" " 499 for sfile in rDict: 500 print(sfile) 501 sortkeys=list(rDict[sfile].keys()) 502 sortkeys.sort() 503 for runex in sortkeys: 504 print(indent+runex) 505 if type(rDict[sfile][runex])==bytes: 506 print(indent*2+rDict[sfile][runex]) 507 else: 508 for var in rDict[sfile][runex]: 509 if var.startswith("test"): continue 510 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 511 if 'subtests' in rDict[sfile][runex]: 512 for var in rDict[sfile][runex]['subtests']: 513 print(indent*2+var) 514 for var2 in rDict[sfile][runex][var]: 515 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 516 print("\n") 517 return 518 519def main(directory='',test_file='',verbosity=0): 520 521 if directory: 522 tDict=parseTestDir(directory,verbosity) 523 else: 524 tDict=parseTestFile(test_file,verbosity) 525 if verbosity>0: printExParseDict(tDict) 526 527 return 528 529if __name__ == '__main__': 530 import optparse 531 parser = optparse.OptionParser() 532 parser.add_option('-d', '--directory', dest='directory', 533 default="", help='Directory containing files to parse') 534 parser.add_option('-t', '--test_file', dest='test_file', 535 default="", help='Test file, e.g., ex1.c, to parse') 536 parser.add_option('-v', '--verbosity', dest='verbosity', 537 help='Verbosity of output by level: 1, 2, or 3', default=0) 538 opts, extra_args = parser.parse_args() 539 540 if extra_args: 541 import sys 542 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 543 exit(1) 544 if not opts.test_file and not opts.directory: 545 print("test file or directory is required") 546 parser.print_usage() 547 sys.exit() 548 549 # Need verbosity to be an integer 550 try: 551 verbosity=int(opts.verbosity) 552 except: 553 raise Exception("Error: Verbosity must be integer") 554 555 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 556