1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args requires comments".split() 54 55import re 56 57def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 58 """ 59 Go through and remove a level of indentation 60 Also strip of trailing whitespace 61 """ 62 # The first entry should be test: but it might be indented. 63 ext=os.path.splitext(srcfile)[1] 64 stripstr=" " 65 if len(fileNums)>0: lineNum=fileNums[0]-1 66 for lline in block.split("\n"): 67 if len(fileNums)>0: lineNum+=1 68 line=lline[1:] if lline.startswith("!") else lline 69 if not line.strip(): continue 70 if line.strip().startswith('#'): continue 71 if entireBlock: 72 var=line.split(":")[0].strip() 73 if not var in ['test','testset','build']: 74 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 75 nspace=len(line)-len(line.lstrip(stripstr)) 76 newline=line[nspace:] 77 break 78 79 # Strip off any indentation for the whole string and any trailing 80 # whitespace for convenience 81 newTestStr="\n" 82 if len(fileNums)>0: lineNum=fileNums[0]-1 83 firstPass=True 84 for lline in block.split("\n"): 85 if len(fileNums)>0: lineNum+=1 86 line=lline[1:] if lline.startswith("!") else lline 87 if not line.strip(): continue 88 if line.strip().startswith('#'): 89 newTestStr+=line+'\n' 90 else: 91 newline=line[nspace:] 92 newTestStr+=newline.rstrip()+"\n" 93 # Do some basic indentation checks 94 if entireBlock: 95 # Don't need to check comment lines 96 if line.strip().startswith('#'): continue 97 if not newline.startswith(" "): 98 var=newline.split(":")[0].strip() 99 if not var in ['test','testset','build']: 100 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 101 if len(fileNums)>0: 102 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 103 else: 104 raise Exception(err) 105 else: 106 var=line.split(":")[0].strip() 107 if var in ['test','testset','build']: 108 subnspace=len(line)-len(line.lstrip(stripstr)) 109 if firstPass: 110 firstsubnspace=subnspace 111 firstPass=False 112 else: 113 if firstsubnspace!=subnspace: 114 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 115 if len(fileNums)>0: 116 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 117 else: 118 raise Exception(err) 119 120 # Allow line continuation character '\' 121 return newTestStr.replace('\\\n', ' ') 122 123def parseLoopArgs(varset): 124 """ 125 Given: String containing loop variables 126 Return: tuple containing separate/shared and string of loop vars 127 """ 128 keynm=varset.split("{{")[0].strip().lstrip('-') 129 if not keynm.strip(): keynm='nsize' 130 lvars=varset.split('{{')[1].split('}')[0] 131 suffx=varset.split('{{')[1].split('}')[1] 132 ftype='separate' if suffx.startswith('separate') else 'shared' 133 return keynm,lvars,ftype 134 135def _getSeparateTestvars(testDict): 136 """ 137 Given: dictionary that may have 138 Return: Variables that cause a test split 139 """ 140 vals=None 141 sepvars=[] 142 # Check nsize 143 if 'nsize' in testDict: 144 varset=testDict['nsize'] 145 if '{{' in varset: 146 keynm,lvars,ftype=parseLoopArgs(varset) 147 if ftype=='separate': sepvars.append(keynm) 148 149 # Now check args 150 if 'args' not in testDict: return sepvars 151 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 152 if not varset.strip(): continue 153 if '{{' in varset: 154 # Assuming only one for loop per var specification 155 keynm,lvars,ftype=parseLoopArgs(varset) 156 if ftype=='separate': sepvars.append(keynm) 157 158 return sepvars 159 160def _getNewArgs(args): 161 """ 162 Given: String that has args that might have loops in them 163 Return: All of the arguments/values that do not have 164 for 'separate output' in for loops 165 """ 166 newargs='' 167 if not args.strip(): return args 168 for varset in re.split('-(?=[a-zA-Z])',args): 169 if not varset.strip(): continue 170 if '{{' not in varset: 171 if 'separate' not in varset: 172 newargs+="-"+varset.strip()+" " 173 174 return newargs 175 176def _getVarVals(findvar,testDict): 177 """ 178 Given: variable that is either nsize or in args 179 Return: Values to loop over and the other arguments 180 Note that we keep the other arguments even if they have 181 for loops to enable stepping through all of the for lops 182 """ 183 save_vals=None 184 if findvar=='nsize': 185 varset=testDict[findvar] 186 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 187 else: 188 varlist=[] 189 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 190 if not varset.strip(): continue 191 if '{{' not in varset: continue 192 keyvar,vals,ftype=parseLoopArgs(varset) 193 if keyvar==findvar: 194 save_vals=vals 195 196 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 197 return save_vals 198 199def genTestsSeparateTestvars(intests,indicts): 200 """ 201 Given: testname, sdict with 'separate_testvars 202 Return: testnames,sdicts: List of generated tests 203 The tricky part here is the {{ ... }separate output} 204 that can be used multiple times 205 """ 206 testnames=[]; sdicts=[] 207 for i in range(len(intests)): 208 testname=intests[i]; sdict=indicts[i]; i+=1 209 separate_testvars=_getSeparateTestvars(sdict) 210 if len(separate_testvars)>0: 211 sep_dicts=[sdict.copy()] 212 if 'args' in sep_dicts[0]: 213 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 214 sep_testnames=[testname] 215 for kvar in separate_testvars: 216 kvals=_getVarVals(kvar,sdict) 217 218 # Have to do loop over previous var/val combos as well 219 # and accumulate as we go 220 val_testnames=[]; val_dicts=[] 221 for val in kvals.split(): 222 gensuffix="_"+kvar+"-"+val.replace(',','__') 223 for kvaltestnm in sep_testnames: 224 val_testnames.append(kvaltestnm+gensuffix) 225 for kv in sep_dicts: 226 kvardict=kv.copy() 227 # If the last var then we have the final version 228 if 'suffix' in sdict: 229 kvardict['suffix']+=gensuffix 230 else: 231 kvardict['suffix']=gensuffix 232 if kvar=='nsize': 233 kvardict[kvar]=val 234 else: 235 kvardict['args']+="-"+kvar+" "+val+" " 236 val_dicts.append(kvardict) 237 sep_testnames=val_testnames 238 sep_dicts=val_dicts 239 testnames+=sep_testnames 240 sdicts+=sep_dicts 241 else: 242 testnames.append(testname) 243 sdicts.append(sdict) 244 return testnames,sdicts 245 246def genTestsSubtestSuffix(testnames,sdicts): 247 """ 248 Given: testname, sdict with separate_testvars 249 Return: testnames,sdicts: List of generated tests 250 """ 251 tnms=[]; sdcts=[] 252 for i in range(len(testnames)): 253 testname=testnames[i] 254 rmsubtests=[]; keepSubtests=False 255 if 'subtests' in sdicts[i]: 256 for stest in sdicts[i]["subtests"]: 257 if 'suffix' in sdicts[i][stest]: 258 rmsubtests.append(stest) 259 gensuffix="_"+sdicts[i][stest]['suffix'] 260 newtestnm=testname+gensuffix 261 tnms.append(newtestnm) 262 newsdict=sdicts[i].copy() 263 del newsdict['subtests'] 264 # Have to hand update 265 # Append 266 for kup in appendlist: 267 if kup in sdicts[i][stest]: 268 if kup in sdicts[i]: 269 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 270 else: 271 newsdict[kup]=sdicts[i][stest][kup] 272 # Promote 273 for kup in acceptedkeys: 274 if kup in appendlist: continue 275 if kup in sdicts[i][stest]: 276 newsdict[kup]=sdicts[i][stest][kup] 277 # Cleanup 278 for st in sdicts[i]["subtests"]: del newsdict[st] 279 sdcts.append(newsdict) 280 else: 281 keepSubtests=True 282 else: 283 tnms.append(testnames[i]) 284 sdcts.append(sdicts[i]) 285 # If a subtest without a suffix exists, then save it 286 if keepSubtests: 287 tnms.append(testnames[i]) 288 newsdict=sdicts[i].copy() 289 # Prune the tests to prepare for keeping 290 for rmtest in rmsubtests: 291 newsdict['subtests'].remove(rmtest) 292 del newsdict[rmtest] 293 sdcts.append(newsdict) 294 i+=1 295 return tnms,sdcts 296 297def splitTests(testname,sdict): 298 """ 299 Given: testname and YAML-generated dictionary 300 Return: list of names and dictionaries corresponding to each test 301 given that the YAML language allows for multiple tests 302 """ 303 304 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 305 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 306 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 307 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts) 308 309 # Because I am altering the list, I do this in passes. Inelegant 310 311 return testnames, sdicts 312 313def parseTest(testStr,srcfile,verbosity): 314 """ 315 This parses an individual test 316 YAML is hierarchial so should use a state machine in the general case, 317 but in practice we only support two levels of test: 318 """ 319 basename=os.path.basename(srcfile) 320 # Handle the new at the begininng 321 bn=re.sub("new_","",basename) 322 # This is the default 323 testname="run"+os.path.splitext(bn)[0] 324 325 # Tests that have default everything (so empty effectively) 326 if len(testStr)==0: return [testname], [{}] 327 328 striptest=_stripIndent(testStr,srcfile) 329 330 # go through and parse 331 subtestnum=0 332 subdict={} 333 comments=[] 334 indentlevel=0 335 for ln in striptest.split("\n"): 336 line=ln.split('#')[0].rstrip() 337 if verbosity>2: print(line) 338 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 339 if comment: comments.append(comment) 340 if not line.strip(): continue 341 lsplit=line.split(':') 342 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 343 indentcount=lsplit[0].count(" ") 344 var=lsplit[0].strip() 345 val=line[line.find(':')+1:].strip() 346 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 347 # Start by seeing if we are in a subtest 348 if line.startswith(" "): 349 if var in subdict[subtestname]: 350 subdict[subtestname][var]+=" "+val 351 else: 352 subdict[subtestname][var]=val 353 if not indentlevel: indentlevel=indentcount 354 #if indentlevel!=indentcount: print("Error in indentation:", ln) 355 # Determine subtest name and make dict 356 elif var=="test": 357 subtestname="test"+str(subtestnum) 358 subdict[subtestname]={} 359 if "subtests" not in subdict: subdict["subtests"]=[] 360 subdict["subtests"].append(subtestname) 361 subtestnum=subtestnum+1 362 # The rest are easy 363 else: 364 # For convenience, it is sometimes convenient to list twice 365 if var in subdict: 366 if var in appendlist: 367 subdict[var]+=" "+val 368 else: 369 raise Exception(var+" entered twice: "+line) 370 else: 371 subdict[var]=val 372 if var=="suffix": 373 if len(val)>0: 374 testname=testname+"_"+val 375 376 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 377 # A test block can create multiple tests. This does 378 # that logic 379 testnames,subdicts=splitTests(testname,subdict) 380 return testnames,subdicts 381 382def parseTests(testStr,srcfile,fileNums,verbosity): 383 """ 384 Parse the yaml string describing tests and return 385 a dictionary with the info in the form of: 386 testDict[test][subtest] 387 This is an inelegant parser as we do not wish to 388 introduce a new dependency by bringing in pyyaml. 389 The advantage is that validation can be done as 390 it is parsed (e.g., 'test' is the only top-level node) 391 """ 392 393 testDict={} 394 395 # The first entry should be test: but it might be indented. 396 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 397 if verbosity>2: print(srcfile) 398 399 ## Check and see if we have build requirements 400 addToRunRequirements=None 401 if "\nbuild:" in newTestStr: 402 testDict['build']={} 403 # The file info is already here and need to append 404 Part1=newTestStr.split("build:")[1] 405 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 406 for bkey in buildkeys: 407 if bkey+":" in fileInfo: 408 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 409 #if verbosity>1: bkey+": "+testDict['build'][bkey] 410 # If a runtime requires are put into build, push them down to all run tests 411 # At this point, we are working with strings and not lists 412 if 'requires' in testDict['build']: 413 if 'datafilespath' in testDict['build']['requires']: 414 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 415 testDict['build']['requires']=newreqs.strip() 416 addToRunRequirements='datafilespath' 417 418 419 # Now go through each test. First elem in split is blank 420 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 421 testnames,subdicts=parseTest(test,srcfile,verbosity) 422 for i in range(len(testnames)): 423 if testnames[i] in testDict: 424 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 425 # Add in build requirements that need to be moved 426 if addToRunRequirements: 427 if 'requires' in subdicts[i]: 428 subdicts[i]['requires']+=addToRunRequirements 429 else: 430 subdicts[i]['requires']=addToRunRequirements 431 testDict[testnames[i]]=subdicts[i] 432 433 return testDict 434 435def parseTestFile(srcfile,verbosity): 436 """ 437 Parse single example files and return dictionary of the form: 438 testDict[srcfile][test][subtest] 439 """ 440 debug=False 441 basename=os.path.basename(srcfile) 442 if basename=='makefile': return {} 443 444 curdir=os.path.realpath(os.path.curdir) 445 basedir=os.path.dirname(os.path.realpath(srcfile)) 446 os.chdir(basedir) 447 448 testDict={} 449 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 450 451 ## Start with doing the tests 452 # 453 fsplit=fileStr.split("/*TEST\n")[1:] 454 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 455 # Allow for multiple "/*TEST" blocks even though it really should be 456 # one 457 srcTests=[] 458 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 459 testString=" ".join(srcTests) 460 flen=len(testString.split("\n")) 461 fend=fstart+flen-1 462 fileNums=range(fstart,fend) 463 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 464 # Massage dictionary for build requirements 465 if 'build' in testDict[basename]: 466 testDict[basename].update(testDict[basename]['build']) 467 del testDict[basename]['build'] 468 469 470 os.chdir(curdir) 471 return testDict 472 473def parseTestDir(directory,verbosity): 474 """ 475 Parse single example files and return dictionary of the form: 476 testDict[srcfile][test][subtest] 477 """ 478 curdir=os.path.realpath(os.path.curdir) 479 basedir=os.path.realpath(directory) 480 os.chdir(basedir) 481 482 tDict={} 483 for test_file in sorted(glob.glob("new_ex*.*")): 484 tDict.update(parseTestFile(test_file,verbosity)) 485 486 os.chdir(curdir) 487 return tDict 488 489def printExParseDict(rDict): 490 """ 491 This is useful for debugging 492 """ 493 indent=" " 494 for sfile in rDict: 495 print(sfile) 496 sortkeys=rDict[sfile].keys() 497 sortkeys.sort() 498 for runex in sortkeys: 499 print(indent+runex) 500 if type(rDict[sfile][runex])==bytes: 501 print(indent*2+rDict[sfile][runex]) 502 else: 503 for var in rDict[sfile][runex]: 504 if var.startswith("test"): continue 505 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 506 if 'subtests' in rDict[sfile][runex]: 507 for var in rDict[sfile][runex]['subtests']: 508 print(indent*2+var) 509 for var2 in rDict[sfile][runex][var]: 510 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 511 print("\n") 512 return 513 514def main(directory='',test_file='',verbosity=0): 515 516 if directory: 517 tDict=parseTestDir(directory,verbosity) 518 else: 519 tDict=parseTestFile(test_file,verbosity) 520 if verbosity>0: printExParseDict(tDict) 521 522 return 523 524if __name__ == '__main__': 525 import optparse 526 parser = optparse.OptionParser() 527 parser.add_option('-d', '--directory', dest='directory', 528 default="", help='Directory containing files to parse') 529 parser.add_option('-t', '--test_file', dest='test_file', 530 default="", help='Test file, e.g., ex1.c, to parse') 531 parser.add_option('-v', '--verbosity', dest='verbosity', 532 help='Verbosity of output by level: 1, 2, or 3', default=0) 533 opts, extra_args = parser.parse_args() 534 535 if extra_args: 536 import sys 537 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 538 exit(1) 539 if not opts.test_file and not opts.directory: 540 print("test file or directory is required") 541 parser.print_usage() 542 sys.exit() 543 544 # Need verbosity to be an integer 545 try: 546 verbosity=int(opts.verbosity) 547 except: 548 raise Exception("Error: Verbosity must be integer") 549 550 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 551