1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args requires comments".split() 54 55import re 56 57def getDefaultOutputFileRoot(testname): 58 """ 59 Given testname, give DefaultRoot and DefaultOutputFilename 60 e.g., runex1 gives ex1_1, output/ex1_1.out 61 """ 62 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 63 if not "_" in defroot: defroot=defroot+"_1" 64 return defroot 65 66def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 67 """ 68 Go through and remove a level of indentation 69 Also strip of trailing whitespace 70 """ 71 # The first entry should be test: but it might be indented. 72 ext=os.path.splitext(srcfile)[1] 73 stripstr=" " 74 if len(fileNums)>0: lineNum=fileNums[0]-1 75 for lline in block.split("\n"): 76 if len(fileNums)>0: lineNum+=1 77 line=lline[1:] if lline.startswith("!") else lline 78 if not line.strip(): continue 79 if line.strip().startswith('#'): continue 80 if entireBlock: 81 var=line.split(":")[0].strip() 82 if not var in ['test','testset','build']: 83 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 84 nspace=len(line)-len(line.lstrip(stripstr)) 85 newline=line[nspace:] 86 break 87 88 # Strip off any indentation for the whole string and any trailing 89 # whitespace for convenience 90 newTestStr="\n" 91 if len(fileNums)>0: lineNum=fileNums[0]-1 92 firstPass=True 93 for lline in block.split("\n"): 94 if len(fileNums)>0: lineNum+=1 95 line=lline[1:] if lline.startswith("!") else lline 96 if not line.strip(): continue 97 if line.strip().startswith('#'): 98 newTestStr+=line+'\n' 99 else: 100 newline=line[nspace:] 101 newTestStr+=newline.rstrip()+"\n" 102 # Do some basic indentation checks 103 if entireBlock: 104 # Don't need to check comment lines 105 if line.strip().startswith('#'): continue 106 if not newline.startswith(" "): 107 var=newline.split(":")[0].strip() 108 if not var in ['test','testset','build']: 109 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 110 if len(fileNums)>0: 111 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 112 else: 113 raise Exception(err) 114 else: 115 var=line.split(":")[0].strip() 116 if var in ['test','testset','build']: 117 subnspace=len(line)-len(line.lstrip(stripstr)) 118 if firstPass: 119 firstsubnspace=subnspace 120 firstPass=False 121 else: 122 if firstsubnspace!=subnspace: 123 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 124 if len(fileNums)>0: 125 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 126 else: 127 raise Exception(err) 128 129 # Allow line continuation character '\' 130 return newTestStr.replace('\\\n', ' ') 131 132def parseLoopArgs(varset): 133 """ 134 Given: String containing loop variables 135 Return: tuple containing separate/shared and string of loop vars 136 """ 137 keynm=varset.split("{{")[0].strip().lstrip('-') 138 if not keynm.strip(): keynm='nsize' 139 lvars=varset.split('{{')[1].split('}')[0] 140 suffx=varset.split('{{')[1].split('}')[1] 141 ftype='separate' if suffx.startswith('separate') else 'shared' 142 return keynm,lvars,ftype 143 144def _getSeparateTestvars(testDict): 145 """ 146 Given: dictionary that may have 147 Return: Variables that cause a test split 148 """ 149 vals=None 150 sepvars=[] 151 # Check nsize 152 if 'nsize' in testDict: 153 varset=testDict['nsize'] 154 if '{{' in varset: 155 keynm,lvars,ftype=parseLoopArgs(varset) 156 if ftype=='separate': sepvars.append(keynm) 157 158 # Now check args 159 if 'args' not in testDict: return sepvars 160 for varset in re.split('(^|\W)-(?=[a-zA-Z])',testDict['args']): 161 if not varset.strip(): continue 162 if '{{' in varset: 163 # Assuming only one for loop per var specification 164 keynm,lvars,ftype=parseLoopArgs(varset) 165 if ftype=='separate': sepvars.append(keynm) 166 167 return sepvars 168 169def _getNewArgs(args): 170 """ 171 Given: String that has args that might have loops in them 172 Return: All of the arguments/values that do not have 173 for 'separate output' in for loops 174 """ 175 newargs='' 176 if not args.strip(): return args 177 for varset in re.split('(^|\W)-(?=[a-zA-Z])',args): 178 if not varset.strip(): continue 179 if '{{' not in varset: 180 if 'separate' not in varset: 181 newargs+="-"+varset.strip()+" " 182 183 return newargs 184 185def _getVarVals(findvar,testDict): 186 """ 187 Given: variable that is either nsize or in args 188 Return: Values to loop over and the other arguments 189 Note that we keep the other arguments even if they have 190 for loops to enable stepping through all of the for lops 191 """ 192 save_vals=None 193 if findvar=='nsize': 194 varset=testDict[findvar] 195 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 196 else: 197 varlist=[] 198 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 199 if not varset.strip(): continue 200 if '{{' not in varset: continue 201 keyvar,vals,ftype=parseLoopArgs(varset) 202 if keyvar==findvar: 203 save_vals=vals 204 205 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 206 return save_vals 207 208def genTestsSeparateTestvars(intests,indicts,final=False): 209 """ 210 Given: testname, sdict with 'separate_testvars 211 Return: testnames,sdicts: List of generated tests 212 The tricky part here is the {{ ... }separate output} 213 that can be used multiple times 214 """ 215 testnames=[]; sdicts=[] 216 for i in range(len(intests)): 217 testname=intests[i]; sdict=indicts[i]; i+=1 218 separate_testvars=_getSeparateTestvars(sdict) 219 if len(separate_testvars)>0: 220 sep_dicts=[sdict.copy()] 221 if 'args' in sep_dicts[0]: 222 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 223 sep_testnames=[testname] 224 for kvar in separate_testvars: 225 kvals=_getVarVals(kvar,sdict) 226 227 # Have to do loop over previous var/val combos as well 228 # and accumulate as we go 229 val_testnames=[]; val_dicts=[] 230 for val in kvals.split(): 231 gensuffix="_"+kvar+"-"+val.replace(',','__') 232 for kvaltestnm in sep_testnames: 233 val_testnames.append(kvaltestnm+gensuffix) 234 for kv in sep_dicts: 235 kvardict=kv.copy() 236 # If the last var then we have the final version 237 if 'suffix' in sdict: 238 kvardict['suffix']+=gensuffix 239 else: 240 kvardict['suffix']=gensuffix 241 if kvar=='nsize': 242 kvardict[kvar]=val 243 else: 244 kvardict['args']+="-"+kvar+" "+val+" " 245 val_dicts.append(kvardict) 246 sep_testnames=val_testnames 247 sep_dicts=val_dicts 248 testnames+=sep_testnames 249 sdicts+=sep_dicts 250 else: 251 # These are plain vanilla tests (no subtests, no loops) that 252 # do not have a suffix. This makes the targets match up with 253 # the output file (testname_1.out) 254 if final: 255 if '_' not in testname: testname+='_1' 256 testnames.append(testname) 257 sdicts.append(sdict) 258 return testnames,sdicts 259 260def genTestsSubtestSuffix(testnames,sdicts): 261 """ 262 Given: testname, sdict with separate_testvars 263 Return: testnames,sdicts: List of generated tests 264 """ 265 tnms=[]; sdcts=[] 266 for i in range(len(testnames)): 267 testname=testnames[i] 268 rmsubtests=[]; keepSubtests=False 269 if 'subtests' in sdicts[i]: 270 for stest in sdicts[i]["subtests"]: 271 if 'suffix' in sdicts[i][stest]: 272 rmsubtests.append(stest) 273 gensuffix="_"+sdicts[i][stest]['suffix'] 274 newtestnm=testname+gensuffix 275 tnms.append(newtestnm) 276 newsdict=sdicts[i].copy() 277 del newsdict['subtests'] 278 # Have to hand update 279 # Append 280 for kup in appendlist: 281 if kup in sdicts[i][stest]: 282 if kup in sdicts[i]: 283 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 284 else: 285 newsdict[kup]=sdicts[i][stest][kup] 286 # Promote 287 for kup in acceptedkeys: 288 if kup in appendlist: continue 289 if kup in sdicts[i][stest]: 290 newsdict[kup]=sdicts[i][stest][kup] 291 # Cleanup 292 for st in sdicts[i]["subtests"]: del newsdict[st] 293 sdcts.append(newsdict) 294 else: 295 keepSubtests=True 296 else: 297 tnms.append(testnames[i]) 298 sdcts.append(sdicts[i]) 299 # If a subtest without a suffix exists, then save it 300 if keepSubtests: 301 tnms.append(testnames[i]) 302 newsdict=sdicts[i].copy() 303 # Prune the tests to prepare for keeping 304 for rmtest in rmsubtests: 305 newsdict['subtests'].remove(rmtest) 306 del newsdict[rmtest] 307 sdcts.append(newsdict) 308 i+=1 309 return tnms,sdcts 310 311def splitTests(testname,sdict): 312 """ 313 Given: testname and YAML-generated dictionary 314 Return: list of names and dictionaries corresponding to each test 315 given that the YAML language allows for multiple tests 316 """ 317 318 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 319 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 320 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 321 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 322 323 # Because I am altering the list, I do this in passes. Inelegant 324 325 return testnames, sdicts 326 327def parseTest(testStr,srcfile,verbosity): 328 """ 329 This parses an individual test 330 YAML is hierarchial so should use a state machine in the general case, 331 but in practice we only support two levels of test: 332 """ 333 basename=os.path.basename(srcfile) 334 # Handle the new at the begininng 335 bn=re.sub("new_","",basename) 336 # This is the default 337 testname="run"+os.path.splitext(bn)[0] 338 339 # Tests that have default everything (so empty effectively) 340 if len(testStr)==0: return [testname], [{}] 341 342 striptest=_stripIndent(testStr,srcfile) 343 344 # go through and parse 345 subtestnum=0 346 subdict={} 347 comments=[] 348 indentlevel=0 349 for ln in striptest.split("\n"): 350 line=ln.split('#')[0].rstrip() 351 if verbosity>2: print(line) 352 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 353 if comment: comments.append(comment) 354 if not line.strip(): continue 355 lsplit=line.split(':') 356 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 357 indentcount=lsplit[0].count(" ") 358 var=lsplit[0].strip() 359 val=line[line.find(':')+1:].strip() 360 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 361 # Start by seeing if we are in a subtest 362 if line.startswith(" "): 363 if var in subdict[subtestname]: 364 subdict[subtestname][var]+=" "+val 365 else: 366 subdict[subtestname][var]=val 367 if not indentlevel: indentlevel=indentcount 368 #if indentlevel!=indentcount: print("Error in indentation:", ln) 369 # Determine subtest name and make dict 370 elif var=="test": 371 subtestname="test"+str(subtestnum) 372 subdict[subtestname]={} 373 if "subtests" not in subdict: subdict["subtests"]=[] 374 subdict["subtests"].append(subtestname) 375 subtestnum=subtestnum+1 376 # The rest are easy 377 else: 378 # For convenience, it is sometimes convenient to list twice 379 if var in subdict: 380 if var in appendlist: 381 subdict[var]+=" "+val 382 else: 383 raise Exception(var+" entered twice: "+line) 384 else: 385 subdict[var]=val 386 if var=="suffix": 387 if len(val)>0: 388 testname+="_"+val 389 390 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 391 392 # A test block can create multiple tests. This does that logic 393 testnames,subdicts=splitTests(testname,subdict) 394 return testnames,subdicts 395 396def parseTests(testStr,srcfile,fileNums,verbosity): 397 """ 398 Parse the yaml string describing tests and return 399 a dictionary with the info in the form of: 400 testDict[test][subtest] 401 This is an inelegant parser as we do not wish to 402 introduce a new dependency by bringing in pyyaml. 403 The advantage is that validation can be done as 404 it is parsed (e.g., 'test' is the only top-level node) 405 """ 406 407 testDict={} 408 409 # The first entry should be test: but it might be indented. 410 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 411 if verbosity>2: print(srcfile) 412 413 ## Check and see if we have build requirements 414 addToRunRequirements=None 415 if "\nbuild:" in newTestStr: 416 testDict['build']={} 417 # The file info is already here and need to append 418 Part1=newTestStr.split("build:")[1] 419 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 420 for bkey in buildkeys: 421 if bkey+":" in fileInfo: 422 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 423 #if verbosity>1: bkey+": "+testDict['build'][bkey] 424 # If a runtime requires are put into build, push them down to all run tests 425 # At this point, we are working with strings and not lists 426 if 'requires' in testDict['build']: 427 if 'datafilespath' in testDict['build']['requires']: 428 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 429 testDict['build']['requires']=newreqs.strip() 430 addToRunRequirements='datafilespath' 431 432 433 # Now go through each test. First elem in split is blank 434 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 435 testnames,subdicts=parseTest(test,srcfile,verbosity) 436 for i in range(len(testnames)): 437 if testnames[i] in testDict: 438 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 439 # Add in build requirements that need to be moved 440 if addToRunRequirements: 441 if 'requires' in subdicts[i]: 442 subdicts[i]['requires']+=addToRunRequirements 443 else: 444 subdicts[i]['requires']=addToRunRequirements 445 testDict[testnames[i]]=subdicts[i] 446 447 return testDict 448 449def parseTestFile(srcfile,verbosity): 450 """ 451 Parse single example files and return dictionary of the form: 452 testDict[srcfile][test][subtest] 453 """ 454 debug=False 455 basename=os.path.basename(srcfile) 456 if basename=='makefile': return {} 457 458 curdir=os.path.realpath(os.path.curdir) 459 basedir=os.path.dirname(os.path.realpath(srcfile)) 460 os.chdir(basedir) 461 462 testDict={} 463 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 464 465 ## Start with doing the tests 466 # 467 fsplit=fileStr.split("/*TEST\n")[1:] 468 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 469 # Allow for multiple "/*TEST" blocks even though it really should be 470 # one 471 srcTests=[] 472 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 473 testString=" ".join(srcTests) 474 flen=len(testString.split("\n")) 475 fend=fstart+flen-1 476 fileNums=range(fstart,fend) 477 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 478 # Massage dictionary for build requirements 479 if 'build' in testDict[basename]: 480 testDict[basename].update(testDict[basename]['build']) 481 del testDict[basename]['build'] 482 483 484 os.chdir(curdir) 485 return testDict 486 487def parseTestDir(directory,verbosity): 488 """ 489 Parse single example files and return dictionary of the form: 490 testDict[srcfile][test][subtest] 491 """ 492 curdir=os.path.realpath(os.path.curdir) 493 basedir=os.path.realpath(directory) 494 os.chdir(basedir) 495 496 tDict={} 497 for test_file in sorted(glob.glob("new_ex*.*")): 498 tDict.update(parseTestFile(test_file,verbosity)) 499 500 os.chdir(curdir) 501 return tDict 502 503def printExParseDict(rDict): 504 """ 505 This is useful for debugging 506 """ 507 indent=" " 508 for sfile in rDict: 509 print(sfile) 510 sortkeys=list(rDict[sfile].keys()) 511 sortkeys.sort() 512 for runex in sortkeys: 513 print(indent+runex) 514 if type(rDict[sfile][runex])==bytes: 515 print(indent*2+rDict[sfile][runex]) 516 else: 517 for var in rDict[sfile][runex]: 518 if var.startswith("test"): continue 519 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 520 if 'subtests' in rDict[sfile][runex]: 521 for var in rDict[sfile][runex]['subtests']: 522 print(indent*2+var) 523 for var2 in rDict[sfile][runex][var]: 524 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 525 print("\n") 526 return 527 528def main(directory='',test_file='',verbosity=0): 529 530 if directory: 531 tDict=parseTestDir(directory,verbosity) 532 else: 533 tDict=parseTestFile(test_file,verbosity) 534 if verbosity>0: printExParseDict(tDict) 535 536 return 537 538if __name__ == '__main__': 539 import optparse 540 parser = optparse.OptionParser() 541 parser.add_option('-d', '--directory', dest='directory', 542 default="", help='Directory containing files to parse') 543 parser.add_option('-t', '--test_file', dest='test_file', 544 default="", help='Test file, e.g., ex1.c, to parse') 545 parser.add_option('-v', '--verbosity', dest='verbosity', 546 help='Verbosity of output by level: 1, 2, or 3', default=0) 547 opts, extra_args = parser.parse_args() 548 549 if extra_args: 550 import sys 551 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 552 exit(1) 553 if not opts.test_file and not opts.directory: 554 print("test file or directory is required") 555 parser.print_usage() 556 sys.exit() 557 558 # Need verbosity to be an integer 559 try: 560 verbosity=int(opts.verbosity) 561 except: 562 raise Exception("Error: Verbosity must be integer") 563 564 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 565