1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13Example language 14---------------- 15 16/*TEST 17 build: 18 requires: moab 19 # This is equivalent to test: 20 testset: 21 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 22 23 testset: 24 suffix: 2 25 nsize: 2 26 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 27 28 testset: 29 suffix: 2 30 nsize: 2 31 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 32 test: 33 34TEST*/ 35 36""" 37 38import os, re, glob, types 39import sys 40import logging 41sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 42 43import inspect 44thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 45maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 46sys.path.insert(0,maintdir) 47 48# These are special keys describing build 49buildkeys="requires TODO SKIP depends".split() 50 51acceptedkeys="test nsize requires command suffix args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split() 52appendlist="args requires comments".split() 53 54import re 55 56def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 57 """ 58 Go through and remove a level of indentation 59 Also strip of trailing whitespace 60 """ 61 # The first entry should be test: but it might be indented. 62 ext=os.path.splitext(srcfile)[1] 63 stripstr=" " 64 if len(fileNums)>0: lineNum=fileNums[0]-1 65 for lline in block.split("\n"): 66 if len(fileNums)>0: lineNum+=1 67 line=lline[1:] if lline.startswith("!") else lline 68 if not line.strip(): continue 69 if line.strip().startswith('#'): continue 70 if entireBlock: 71 var=line.split(":")[0].strip() 72 if not var in ['test','testset','build']: 73 raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n") 74 nspace=len(line)-len(line.lstrip(stripstr)) 75 newline=line[nspace:] 76 break 77 78 # Strip off any indentation for the whole string and any trailing 79 # whitespace for convenience 80 newTestStr="\n" 81 if len(fileNums)>0: lineNum=fileNums[0]-1 82 firstPass=True 83 for lline in block.split("\n"): 84 if len(fileNums)>0: lineNum+=1 85 line=lline[1:] if lline.startswith("!") else lline 86 if not line.strip(): continue 87 if line.strip().startswith('#'): 88 newTestStr+=line+'\n' 89 else: 90 newline=line[nspace:] 91 newTestStr+=newline.rstrip()+"\n" 92 # Do some basic indentation checks 93 if entireBlock: 94 # Don't need to check comment lines 95 if line.strip().startswith('#'): continue 96 if not newline.startswith(" "): 97 var=newline.split(":")[0].strip() 98 if not var in ['test','testset','build']: 99 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 100 if len(fileNums)>0: 101 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 102 else: 103 raise Exception(err) 104 else: 105 var=line.split(":")[0].strip() 106 if var in ['test','testset','build']: 107 subnspace=len(line)-len(line.lstrip(stripstr)) 108 if firstPass: 109 firstsubnspace=subnspace 110 firstPass=False 111 else: 112 if firstsubnspace!=subnspace: 113 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 114 if len(fileNums)>0: 115 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 116 else: 117 raise Exception(err) 118 119 120 return newTestStr 121 122def parseLoopArgs(varset): 123 """ 124 Given: String containing loop variables 125 Return: tuple containing separate/shared and string of loop vars 126 """ 127 keynm=varset.split("{{")[0].strip() 128 if not keynm.strip(): keynm='nsize' 129 lvars=varset.split('{{')[1].split('}')[0] 130 suffx=varset.split('{{')[1].split('}')[1] 131 ftype='separate' if suffx.startswith('separate') else 'shared' 132 return keynm,lvars,ftype 133 134def _getSeparateTestvars(testDict): 135 """ 136 Given: dictionary that may have 137 Return: Variables that cause a test split 138 """ 139 vals=None 140 sepvars=[] 141 # Check nsize 142 if testDict.has_key('nsize'): 143 varset=testDict['nsize'] 144 if '{{' in varset: 145 keynm,lvars,ftype=parseLoopArgs(varset) 146 if ftype=='separate': sepvars.append(keynm) 147 148 # Now check args 149 if not testDict.has_key('args'): return sepvars 150 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 151 if not varset.strip(): continue 152 if '{{' in varset: 153 # Assuming only one for loop per var specification 154 keynm,lvars,ftype=parseLoopArgs(varset) 155 if ftype=='separate': sepvars.append(keynm) 156 157 return sepvars 158 159def _getVarVals(findvar,testDict): 160 """ 161 Given: variable that is either nsize or in args 162 Return: Values to loop over 163 """ 164 vals=None 165 newargs='' 166 if findvar=='nsize': 167 varset=testDict[findvar] 168 keynm,vals,ftype=parseLoopArgs('nsize '+varset) 169 else: 170 varlist=[] 171 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 172 if not varset.strip(): continue 173 if '{{' in varset: 174 keyvar,vals,ftype=parseLoopArgs(varset) 175 if keyvar!=findvar: 176 newargs+="-"+varset.strip()+" " 177 continue 178 else: 179 newargs+="-"+varset.strip()+" " 180 181 if not vals: raise StandardError("Could not find separate_testvar: "+findvar) 182 return vals,newargs 183 184def genTestsSeparateTestvars(intests,indicts): 185 """ 186 Given: testname, sdict with 'separate_testvars 187 Return: testnames,sdicts: List of generated tests 188 """ 189 testnames=[]; sdicts=[] 190 for i in range(len(intests)): 191 testname=intests[i]; sdict=indicts[i]; i+=1 192 separate_testvars=_getSeparateTestvars(sdict) 193 if len(separate_testvars)>0: 194 for kvar in separate_testvars: 195 kvals,newargs=_getVarVals(kvar,sdict) 196 # No errors means we are good to go 197 for val in kvals.split(): 198 kvardict=sdict.copy() 199 gensuffix="_"+kvar+"-"+val 200 newtestnm=testname+gensuffix 201 if sdict.has_key('suffix'): 202 kvardict['suffix']=sdict['suffix']+gensuffix 203 else: 204 kvardict['suffix']=gensuffix 205 if kvar=='nsize': 206 kvardict[kvar]=val 207 else: 208 kvardict['args']=newargs+"-"+kvar+" "+val 209 testnames.append(newtestnm) 210 sdicts.append(kvardict) 211 else: 212 testnames.append(testname) 213 sdicts.append(sdict) 214 return testnames,sdicts 215 216def genTestsSubtestSuffix(testnames,sdicts): 217 """ 218 Given: testname, sdict with separate_testvars 219 Return: testnames,sdicts: List of generated tests 220 """ 221 tnms=[]; sdcts=[] 222 for i in range(len(testnames)): 223 testname=testnames[i] 224 rmsubtests=[]; keepSubtests=False 225 if sdicts[i].has_key('subtests'): 226 for stest in sdicts[i]["subtests"]: 227 if sdicts[i][stest].has_key('suffix'): 228 rmsubtests.append(stest) 229 gensuffix="_"+sdicts[i][stest]['suffix'] 230 newtestnm=testname+gensuffix 231 tnms.append(newtestnm) 232 newsdict=sdicts[i].copy() 233 del newsdict['subtests'] 234 # Have to hand update 235 # Append 236 for kup in appendlist: 237 if sdicts[i][stest].has_key(kup): 238 if sdicts[i].has_key(kup): 239 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 240 else: 241 newsdict[kup]=sdicts[i][stest][kup] 242 # Promote 243 for kup in acceptedkeys: 244 if kup in appendlist: continue 245 if sdicts[i][stest].has_key(kup): 246 newsdict[kup]=sdicts[i][stest][kup] 247 # Cleanup 248 for st in sdicts[i]["subtests"]: del newsdict[st] 249 sdcts.append(newsdict) 250 else: 251 keepSubtests=True 252 else: 253 tnms.append(testnames[i]) 254 sdcts.append(sdicts[i]) 255 # If a subtest without a suffix exists, then save it 256 if keepSubtests: 257 tnms.append(testnames[i]) 258 newsdict=sdicts[i].copy() 259 # Prune the tests to prepare for keeping 260 for rmtest in rmsubtests: 261 newsdict['subtests'].remove(rmtest) 262 del newsdict[rmtest] 263 sdcts.append(newsdict) 264 i+=1 265 return tnms,sdcts 266 267def splitTests(testname,sdict): 268 """ 269 Given: testname and YAML-generated dictionary 270 Return: list of names and dictionaries corresponding to each test 271 given that the YAML language allows for multiple tests 272 """ 273 274 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 275 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 276 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 277 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts) 278 279 # Because I am altering the list, I do this in passes. Inelegant 280 281 return testnames, sdicts 282 283def parseTest(testStr,srcfile,verbosity): 284 """ 285 This parses an individual test 286 YAML is hierarchial so should use a state machine in the general case, 287 but in practice we only support two levels of test: 288 """ 289 basename=os.path.basename(srcfile) 290 # Handle the new at the begininng 291 bn=re.sub("new_","",basename) 292 # This is the default 293 testname="run"+os.path.splitext(bn)[0] 294 295 # Tests that have default everything (so empty effectively) 296 if len(testStr)==0: return [testname], [{}] 297 298 striptest=_stripIndent(testStr,srcfile) 299 300 # go through and parse 301 subtestnum=0 302 subdict={} 303 comments=[] 304 indentlevel=0 305 for ln in striptest.split("\n"): 306 line=ln.split('#')[0].rstrip() 307 if verbosity>2: print(line) 308 comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip()) 309 if comment: comments.append(comment) 310 if not line.strip(): continue 311 lsplit=line.split(':') 312 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 313 indentcount=lsplit[0].count(" ") 314 var=lsplit[0].strip() 315 val=line[line.find(':')+1:].strip() 316 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 317 # Start by seeing if we are in a subtest 318 if line.startswith(" "): 319 subdict[subtestname][var]=val 320 if not indentlevel: indentlevel=indentcount 321 #if indentlevel!=indentcount: print("Error in indentation:", ln) 322 # Determine subtest name and make dict 323 elif var=="test": 324 subtestname="test"+str(subtestnum) 325 subdict[subtestname]={} 326 if not subdict.has_key("subtests"): subdict["subtests"]=[] 327 subdict["subtests"].append(subtestname) 328 subtestnum=subtestnum+1 329 # The rest are easy 330 else: 331 # For convenience, it is sometimes convenient to list twice 332 if subdict.has_key(var): 333 if var in appendlist: 334 subdict[var]+=" "+val 335 else: 336 raise Exception(var+" entered twice: "+line) 337 else: 338 subdict[var]=val 339 if var=="suffix": 340 if len(val)>0: 341 testname=testname+"_"+val 342 343 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 344 # A test block can create multiple tests. This does 345 # that logic 346 testnames,subdicts=splitTests(testname,subdict) 347 return testnames,subdicts 348 349def parseTests(testStr,srcfile,fileNums,verbosity): 350 """ 351 Parse the yaml string describing tests and return 352 a dictionary with the info in the form of: 353 testDict[test][subtest] 354 This is an inelegant parser as we do not wish to 355 introduce a new dependency by bringing in pyyaml. 356 The advantage is that validation can be done as 357 it is parsed (e.g., 'test' is the only top-level node) 358 """ 359 360 testDict={} 361 362 # The first entry should be test: but it might be indented. 363 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 364 if verbosity>2: print(srcfile) 365 366 ## Check and see if we have build reuqirements 367 if "\nbuild:" in newTestStr: 368 testDict['build']={} 369 # The file info is already here and need to append 370 Part1=newTestStr.split("build:")[1] 371 fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0] 372 for bkey in buildkeys: 373 if bkey+":" in fileInfo: 374 testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 375 #if verbosity>1: bkey+": "+testDict['build'][bkey] 376 377 # Now go through each test. First elem in split is blank 378 for test in re.split("\ntest(?:set)?:",newTestStr)[1:]: 379 testnames,subdicts=parseTest(test,srcfile,verbosity) 380 for i in range(len(testnames)): 381 if testDict.has_key(testnames[i]): 382 raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile) 383 testDict[testnames[i]]=subdicts[i] 384 385 return testDict 386 387def parseTestFile(srcfile,verbosity): 388 """ 389 Parse single example files and return dictionary of the form: 390 testDict[srcfile][test][subtest] 391 """ 392 debug=False 393 basename=os.path.basename(srcfile) 394 if basename=='makefile': return {} 395 396 curdir=os.path.realpath(os.path.curdir) 397 basedir=os.path.dirname(os.path.realpath(srcfile)) 398 os.chdir(basedir) 399 400 testDict={} 401 sh=open(basename,"r"); fileStr=sh.read(); sh.close() 402 403 ## Start with doing the tests 404 # 405 fsplit=fileStr.split("/*TEST\n")[1:] 406 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 407 # Allow for multiple "/*TEST" blocks even though it really should be 408 # one 409 srcTests=[] 410 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 411 testString=" ".join(srcTests) 412 flen=len(testString.split("\n")) 413 fend=fstart+flen-1 414 fileNums=range(fstart,fend) 415 testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity) 416 # Massage dictionary for build requirements 417 if 'build' in testDict[basename]: 418 testDict[basename].update(testDict[basename]['build']) 419 del testDict[basename]['build'] 420 421 422 os.chdir(curdir) 423 return testDict 424 425def parseTestDir(directory,verbosity): 426 """ 427 Parse single example files and return dictionary of the form: 428 testDict[srcfile][test][subtest] 429 """ 430 curdir=os.path.realpath(os.path.curdir) 431 basedir=os.path.realpath(directory) 432 os.chdir(basedir) 433 434 tDict={} 435 for test_file in glob.glob("new_ex*.*"): 436 tDict.update(parseTestFile(test_file,verbosity)) 437 438 os.chdir(curdir) 439 return tDict 440 441def printExParseDict(rDict): 442 """ 443 This is useful for debugging 444 """ 445 indent=" " 446 for sfile in rDict: 447 print(sfile) 448 sortkeys=rDict[sfile].keys() 449 sortkeys.sort() 450 for runex in sortkeys: 451 print(indent+runex) 452 if type(rDict[sfile][runex])==types.StringType: 453 print(indent*2+rDict[sfile][runex]) 454 else: 455 for var in rDict[sfile][runex]: 456 if var.startswith("test"): continue 457 print(indent*2+var+": "+str(rDict[sfile][runex][var])) 458 if rDict[sfile][runex].has_key('subtests'): 459 for var in rDict[sfile][runex]['subtests']: 460 print(indent*2+var) 461 for var2 in rDict[sfile][runex][var]: 462 print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2])) 463 print("\n") 464 return 465 466def main(directory='',test_file='',verbosity=0): 467 468 if directory: 469 tDict=parseTestDir(directory,verbosity) 470 else: 471 tDict=parseTestFile(test_file,verbosity) 472 if verbosity>0: printExParseDict(tDict) 473 474 return 475 476if __name__ == '__main__': 477 import optparse 478 parser = optparse.OptionParser() 479 parser.add_option('-d', '--directory', dest='directory', 480 default="", help='Directory containing files to parse') 481 parser.add_option('-t', '--test_file', dest='test_file', 482 default="", help='Test file, e.g., ex1.c, to parse') 483 parser.add_option('-v', '--verbosity', dest='verbosity', 484 help='Verbosity of output by level: 1, 2, or 3', default=0) 485 opts, extra_args = parser.parse_args() 486 487 if extra_args: 488 import sys 489 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 490 exit(1) 491 if not opts.test_file and not opts.directory: 492 print("test file or directory is required") 493 parser.print_usage() 494 sys.exit() 495 496 # Need verbosity to be an integer 497 try: 498 verbosity=int(opts.verbosity) 499 except: 500 raise Exception("Error: Verbosity must be integer") 501 502 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 503