1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37from __future__ import print_function 38 39import os, re, glob, types 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 53appendlist="args requires comments".split() 54 55import re 56 57def getDefaultOutputFileRoot(testname): 58 """ 59 Given testname, give DefaultRoot and DefaultOutputFilename 60 e.g., runex1 gives ex1_1, output/ex1_1.out 61 """ 62 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 63 if not "_" in defroot: defroot=defroot+"_1" 64 return defroot 65 66def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 67 """ 68 Go through and remove a level of indentation 69 Also strip of trailing whitespace 70 """ 71 # The first entry should be test: but it might be indented. 72 ext=os.path.splitext(srcfile)[1] 73 stripstr=" " 74 if len(fileNums)>0: lineNum=fileNums[0]-1 75 for lline in block.split("\n"): 76 if len(fileNums)>0: lineNum+=1 77 line=lline[1:] if lline.startswith("!") else lline 78 if not line.strip(): continue 79 if line.strip().startswith('#'): continue 80 if entireBlock: 81 var=line.split(":")[0].strip() 82 if not var in ['test','testset','build']: 83 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 84 nspace=len(line)-len(line.lstrip(stripstr)) 85 newline=line[nspace:] 86 break 87 88 # Strip off any indentation for the whole string and any trailing 89 # whitespace for convenience 90 newTestStr="\n" 91 if len(fileNums)>0: lineNum=fileNums[0]-1 92 firstPass=True 93 for lline in block.split("\n"): 94 if len(fileNums)>0: lineNum+=1 95 line=lline[1:] if lline.startswith("!") else lline 96 if not line.strip(): continue 97 if line.strip().startswith('#'): 98 newTestStr+=line+'\n' 99 else: 100 newline=line[nspace:] 101 newTestStr+=newline.rstrip()+"\n" 102 # Do some basic indentation checks 103 if entireBlock: 104 # Don't need to check comment lines 105 if line.strip().startswith('#'): continue 106 if not newline.startswith(" "): 107 var=newline.split(":")[0].strip() 108 if not var in ['test','testset','build']: 109 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 110 if len(fileNums)>0: 111 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 112 else: 113 raise Exception(err) 114 else: 115 var=line.split(":")[0].strip() 116 if var in ['test','testset','build']: 117 subnspace=len(line)-len(line.lstrip(stripstr)) 118 if firstPass: 119 firstsubnspace=subnspace 120 firstPass=False 121 else: 122 if firstsubnspace!=subnspace: 123 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 124 if len(fileNums)>0: 125 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 126 else: 127 raise Exception(err) 128 129 # Allow line continuation character '\' 130 return newTestStr.replace('\\\n', ' ') 131 132def parseLoopArgs(varset): 133 """ 134 Given: String containing loop variables 135 Return: tuple containing separate/shared and string of loop vars 136 """ 137 keynm=varset.split("{{")[0].strip().lstrip('-') 138 if not keynm.strip(): keynm='nsize' 139 lvars=varset.split('{{')[1].split('}')[0] 140 suffx=varset.split('{{')[1].split('}')[1] 141 ftype='separate' if suffx.startswith('separate') else 'shared' 142 return keynm,lvars,ftype 143 144def _getSeparateTestvars(testDict): 145 """ 146 Given: dictionary that may have 147 Return: Variables that cause a test split 148 """ 149 vals=None 150 sepvars=[] 151 # Check nsize 152 if 'nsize' in testDict: 153 varset=testDict['nsize'] 154 if '{{' in varset: 155 keynm,lvars,ftype=parseLoopArgs(varset) 156 if ftype=='separate': sepvars.append(keynm) 157 158 # Now check args 159 if 'args' not in testDict: return sepvars 160 for varset in re.split('(^|\W)-(?=[a-zA-Z])',testDict['args']): 161 if not varset.strip(): continue 162 if '{{' in varset: 163 # Assuming only one for loop per var specification 164 keynm,lvars,ftype=parseLoopArgs(varset) 165 if ftype=='separate': sepvars.append(keynm) 166 167 return sepvars 168 169def _getNewArgs(args): 170 """ 171 Given: String that has args that might have loops in them 172 Return: All of the arguments/values that do not have 173 for 'separate output' in for loops 174 """ 175 newargs='' 176 if not args.strip(): return args 177 for varset in re.split('(^|\W)-(?=[a-zA-Z])',args): 178 if not varset.strip(): continue 179 if '{{' in varset and 'separate' in varset: continue 180 newargs+="-"+varset.strip()+" " 181 182 return newargs 183 184def _getVarVals(findvar,testDict): 185 """ 186 Given: variable that is either nsize or in args 187 Return: Values to loop over and the other arguments 188 Note that we keep the other arguments even if they have 189 for loops to enable stepping through all of the for lops 190 """ 191 save_vals=None 192 if findvar=='nsize': 193 varset=testDict[findvar] 194 keynm,save_vals,ftype=parseLoopArgs('nsize '+varset) 195 else: 196 varlist=[] 197 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 198 if not varset.strip(): continue 199 if '{{' not in varset: continue 200 keyvar,vals,ftype=parseLoopArgs(varset) 201 if keyvar==findvar: 202 save_vals=vals 203 204 if not save_vals: raise Exception("Could not find separate_testvar: "+findvar) 205 return save_vals 206 207def genTestsSeparateTestvars(intests,indicts,final=False): 208 """ 209 Given: testname, sdict with 'separate_testvars 210 Return: testnames,sdicts: List of generated tests 211 The tricky part here is the {{ ... }separate output} 212 that can be used multiple times 213 """ 214 testnames=[]; sdicts=[] 215 for i in range(len(intests)): 216 testname=intests[i]; sdict=indicts[i]; i+=1 217 separate_testvars=_getSeparateTestvars(sdict) 218 if len(separate_testvars)>0: 219 sep_dicts=[sdict.copy()] 220 if 'args' in sep_dicts[0]: 221 sep_dicts[0]['args']=_getNewArgs(sdict['args']) 222 sep_testnames=[testname] 223 for kvar in separate_testvars: 224 kvals=_getVarVals(kvar,sdict) 225 226 # Have to do loop over previous var/val combos as well 227 # and accumulate as we go 228 val_testnames=[]; val_dicts=[] 229 for val in kvals.split(): 230 gensuffix="_"+kvar+"-"+val.replace(',','__') 231 for kvaltestnm in sep_testnames: 232 val_testnames.append(kvaltestnm+gensuffix) 233 for kv in sep_dicts: 234 kvardict=kv.copy() 235 # If the last var then we have the final version 236 if 'suffix' in sdict: 237 kvardict['suffix']+=gensuffix 238 else: 239 kvardict['suffix']=gensuffix 240 if kvar=='nsize': 241 kvardict[kvar]=val 242 else: 243 kvardict['args']+="-"+kvar+" "+val+" " 244 val_dicts.append(kvardict) 245 sep_testnames=val_testnames 246 sep_dicts=val_dicts 247 testnames+=sep_testnames 248 sdicts+=sep_dicts 249 else: 250 # These are plain vanilla tests (no subtests, no loops) that 251 # do not have a suffix. This makes the targets match up with 252 # the output file (testname_1.out) 253 if final: 254 if '_' not in testname: testname+='_1' 255 testnames.append(testname) 256 sdicts.append(sdict) 257 return testnames,sdicts 258 259def genTestsSubtestSuffix(testnames,sdicts): 260 """ 261 Given: testname, sdict with separate_testvars 262 Return: testnames,sdicts: List of generated tests 263 """ 264 tnms=[]; sdcts=[] 265 for i in range(len(testnames)): 266 testname=testnames[i] 267 rmsubtests=[]; keepSubtests=False 268 if 'subtests' in sdicts[i]: 269 for stest in sdicts[i]["subtests"]: 270 if 'suffix' in sdicts[i][stest]: 271 rmsubtests.append(stest) 272 gensuffix="_"+sdicts[i][stest]['suffix'] 273 newtestnm=testname+gensuffix 274 tnms.append(newtestnm) 275 newsdict=sdicts[i].copy() 276 del newsdict['subtests'] 277 # Have to hand update 278 # Append 279 for kup in appendlist: 280 if kup in sdicts[i][stest]: 281 if kup in sdicts[i]: 282 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 283 else: 284 newsdict[kup]=sdicts[i][stest][kup] 285 # Promote 286 for kup in acceptedkeys: 287 if kup in appendlist: continue 288 if kup in sdicts[i][stest]: 289 newsdict[kup]=sdicts[i][stest][kup] 290 # Cleanup 291 for st in sdicts[i]["subtests"]: del newsdict[st] 292 sdcts.append(newsdict) 293 else: 294 keepSubtests=True 295 else: 296 tnms.append(testnames[i]) 297 sdcts.append(sdicts[i]) 298 # If a subtest without a suffix exists, then save it 299 if keepSubtests: 300 tnms.append(testnames[i]) 301 newsdict=sdicts[i].copy() 302 # Prune the tests to prepare for keeping 303 for rmtest in rmsubtests: 304 newsdict['subtests'].remove(rmtest) 305 del newsdict[rmtest] 306 sdcts.append(newsdict) 307 i+=1 308 return tnms,sdcts 309 310def splitTests(testname,sdict): 311 """ 312 Given: testname and YAML-generated dictionary 313 Return: list of names and dictionaries corresponding to each test 314 given that the YAML language allows for multiple tests 315 """ 316 317 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 318 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 319 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 320 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True) 321 322 # Because I am altering the list, I do this in passes. Inelegant 323 324 return testnames, sdicts 325 326def parseTest(testStr,srcfile,verbosity): 327 """ 328 This parses an individual test 329 YAML is hierarchial so should use a state machine in the general case, 330 but in practice we only support two levels of test: 331 """ 332 basename=os.path.basename(srcfile) 333 # Handle the new at the begininng 334 bn=re.sub("new_","",basename) 335 # This is the default 336 testname="run"+os.path.splitext(bn)[0] 337 338 # Tests that have default everything (so empty effectively) 339 if len(testStr)==0: return [testname], [{}] 340 341 striptest=_stripIndent(testStr,srcfile) 342 343 # go through and parse 344 subtestnum=0 345 subdict={} 346 comments=[] 347 indentlevel=0 348 for ln in striptest.split("\n"): 349 line=ln.split('#')[0].rstrip() 350 if verbosity>2: print(line) 351 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 352 if comment: comments.append(comment) 353 if not line.strip(): continue 354 lsplit=line.split(':') 355 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 356 indentcount=lsplit[0].count(" ") 357 var=lsplit[0].strip() 358 val=line[line.find(':')+1:].strip() 359 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 360 # Start by seeing if we are in a subtest 361 if line.startswith(" "): 362 if var in subdict[subtestname]: 363 subdict[subtestname][var]+=" "+val 364 else: 365 subdict[subtestname][var]=val 366 if not indentlevel: indentlevel=indentcount 367 #if indentlevel!=indentcount: print("Error in indentation:", ln) 368 # Determine subtest name and make dict 369 elif var=="test": 370 subtestname="test"+str(subtestnum) 371 subdict[subtestname]={} 372 if "subtests" not in subdict: subdict["subtests"]=[] 373 subdict["subtests"].append(subtestname) 374 subtestnum=subtestnum+1 375 # The rest are easy 376 else: 377 # For convenience, it is sometimes convenient to list twice 378 if var in subdict: 379 if var in appendlist: 380 subdict[var]+=" "+val 381 else: 382 raise Exception(var+" entered twice: "+line) 383 else: 384 subdict[var]=val 385 if var=="suffix": 386 if len(val)>0: 387 testname+="_"+val 388 389 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 390 391 # A test block can create multiple tests. This does that logic 392 testnames,subdicts=splitTests(testname,subdict) 393 return testnames,subdicts 394 395def parseTests(testStr,srcfile,fileNums,verbosity): 396 """ 397 Parse the yaml string describing tests and return 398 a dictionary with the info in the form of: 399 testDict[test][subtest] 400 This is an inelegant parser as we do not wish to 401 introduce a new dependency by bringing in pyyaml. 402 The advantage is that validation can be done as 403 it is parsed (e.g., 'test' is the only top-level node) 404 """ 405 406 testDict={} 407 408 # The first entry should be test: but it might be indented. 409 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 410 if verbosity>2: print(srcfile) 411 412 ## Check and see if we have build requirements 413 addToRunRequirements=None 414 if "\nbuild:" in newTestStr: 415 testDict['build']={} 416 # The file info is already here and need to append 417 Part1=newTestStr.split("build:")[1] 418 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 419 for bkey in buildkeys: 420 if bkey+":" in fileInfo: 421 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 422 #if verbosity>1: bkey+": "+testDict['build'][bkey] 423 # If a runtime requires are put into build, push them down to all run tests 424 # At this point, we are working with strings and not lists 425 if 'requires' in testDict['build']: 426 if 'datafilespath' in testDict['build']['requires']: 427 newreqs=re.sub('datafilespath','',testDict['build']['requires']) 428 testDict['build']['requires']=newreqs.strip() 429 addToRunRequirements='datafilespath' 430 431 432 # Now go through each test. First elem in split is blank 433 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 434 testnames,subdicts=parseTest(test,srcfile,verbosity) 435 for i in range(len(testnames)): 436 if testnames[i] in testDict: 437 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 438 # Add in build requirements that need to be moved 439 if addToRunRequirements: 440 if 'requires' in subdicts[i]: 441 subdicts[i]['requires']+=addToRunRequirements 442 else: 443 subdicts[i]['requires']=addToRunRequirements 444 testDict[testnames[i]]=subdicts[i] 445 446 return testDict 447 448def parseTestFile(srcfile,verbosity): 449 """ 450 Parse single example files and return dictionary of the form: 451 testDict[srcfile][test][subtest] 452 """ 453 debug=False 454 basename=os.path.basename(srcfile) 455 if basename=='makefile': return {} 456 457 curdir=os.path.realpath(os.path.curdir) 458 basedir=os.path.dirname(os.path.realpath(srcfile)) 459 os.chdir(basedir) 460 461 testDict={} 462 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 463 464 ## Start with doing the tests 465 # 466 fsplit=fileStr.split("/*TEST\n")[1:] 467 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 468 # Allow for multiple "/*TEST" blocks even though it really should be 469 # one 470 srcTests=[] 471 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 472 testString=" ".join(srcTests) 473 flen=len(testString.split("\n")) 474 fend=fstart+flen-1 475 fileNums=range(fstart,fend) 476 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 477 # Massage dictionary for build requirements 478 if 'build' in testDict[basename]: 479 testDict[basename].update(testDict[basename]['build']) 480 del testDict[basename]['build'] 481 482 483 os.chdir(curdir) 484 return testDict 485 486def parseTestDir(directory,verbosity): 487 """ 488 Parse single example files and return dictionary of the form: 489 testDict[srcfile][test][subtest] 490 """ 491 curdir=os.path.realpath(os.path.curdir) 492 basedir=os.path.realpath(directory) 493 os.chdir(basedir) 494 495 tDict={} 496 for test_file in sorted(glob.glob("new_ex*.*")): 497 tDict.update(parseTestFile(test_file,verbosity)) 498 499 os.chdir(curdir) 500 return tDict 501 502def printExParseDict(rDict): 503 """ 504 This is useful for debugging 505 """ 506 indent=" " 507 for sfile in rDict: 508 print(sfile) 509 sortkeys=list(rDict[sfile].keys()) 510 sortkeys.sort() 511 for runex in sortkeys: 512 if runex == 'requires': 513 print(indent+runex+':'+str(rDict[sfile][runex])) 514 continue 515 print(indent+runex) 516 if type(rDict[sfile][runex])==bytes: 517 print(indent*2+rDict[sfile][runex]) 518 else: 519 for var in rDict[sfile][runex]: 520 if var.startswith("test"): continue 521 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 522 if 'subtests' in rDict[sfile][runex]: 523 for var in rDict[sfile][runex]['subtests']: 524 print(indent*2+var) 525 for var2 in rDict[sfile][runex][var]: 526 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 527 print("\n") 528 return 529 530def main(directory='',test_file='',verbosity=0): 531 532 if directory: 533 tDict=parseTestDir(directory,verbosity) 534 else: 535 tDict=parseTestFile(test_file,verbosity) 536 if verbosity>0: printExParseDict(tDict) 537 538 return 539 540if __name__ == '__main__': 541 import optparse 542 parser = optparse.OptionParser() 543 parser.add_option('-d', '--directory', dest='directory', 544 default="", help='Directory containing files to parse') 545 parser.add_option('-t', '--test_file', dest='test_file', 546 default="", help='Test file, e.g., ex1.c, to parse') 547 parser.add_option('-v', '--verbosity', dest='verbosity', 548 help='Verbosity of output by level: 1, 2, or 3', default=0) 549 opts, extra_args = parser.parse_args() 550 551 if extra_args: 552 import sys 553 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 554 exit(1) 555 if not opts.test_file and not opts.directory: 556 print("test file or directory is required") 557 parser.print_usage() 558 sys.exit() 559 560 # Need verbosity to be an integer 561 try: 562 verbosity=int(opts.verbosity) 563 except: 564 raise Exception("Error: Verbosity must be integer") 565 566 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 567