1#!/usr/bin/env python 2""" 3Parse the test file and return a dictionary. 4 5Quick usage:: 6 7 bin/maint/testparse.py -t src/ksp/ksp/examples/tutorials/ex1.c 8 9From the command line, it prints out the dictionary. 10This is meant to be used by other scripts, but it is 11useful to debug individual files. 12 13 14 15Example language 16---------------- 17/*T 18 Concepts: 19 requires: moab 20T*/ 21 22 23 24/*TEST 25 26 test: 27 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 28 29 test: 30 suffix: 2 31 nsize: 2 32 args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full 33 34TEST*/ 35 36""" 37 38import os, re, glob, types 39from distutils.sysconfig import parse_makefile 40import sys 41import logging 42sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 43 44import inspect 45thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 46maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint') 47sys.path.insert(0,maintdir) 48 49# These are special keys describing build 50buildkeys="requires TODO SKIP depends".split() 51 52acceptedkeys="test nsize requires command suffix args separate_testvars filter filter_output localrunfiles comments TODO SKIP output_file".split() 53appendlist="args requires comments".split() 54 55import re 56 57def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]): 58 """ 59 Go through and remove a level of indentation 60 Also strip of trailing whitespace 61 """ 62 # The first entry should be test: but it might be indented. 63 ext=os.path.splitext(srcfile)[1] 64 stripstr=" " 65 for lline in block.split("\n"): 66 line=lline[1:] if lline.startswith("!") else lline 67 if not line.strip(): continue 68 if line.strip().startswith('#'): continue 69 if entireBlock: 70 var=line.split(":")[0].strip() 71 if not var=='test': 72 raise Exception("Formatting error in finding test in file: "+srcfile+"\n") 73 nspace=len(line)-len(line.lstrip(stripstr)) 74 newline=line[nspace:] 75 break 76 77 # Strip off any indentation for the whole string and any trailing 78 # whitespace for convenience 79 newTestStr="\n" 80 if len(fileNums)>0: lineNum=fileNums[0]-1 81 firstPass=True 82 for lline in block.split("\n"): 83 if len(fileNums)>0: lineNum+=1 84 line=lline[1:] if lline.startswith("!") else lline 85 if not line.strip(): continue 86 if line.strip().startswith('#'): 87 newTestStr+=line+'\n' 88 else: 89 newline=line[nspace:] 90 newTestStr+=newline.rstrip()+"\n" 91 # Do some basic indentation checks 92 if entireBlock: 93 # Don't need to check comment lines 94 if line.strip().startswith('#'): continue 95 if not newline.startswith(" "): 96 var=newline.split(":")[0].strip() 97 if not var=='test': 98 err="Formatting error in file "+srcfile+" at line: " +line+"\n" 99 if len(fileNums)>0: 100 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 101 else: 102 raise Exception(err) 103 else: 104 var=line.split(":")[0].strip() 105 if var=='test': 106 subnspace=len(line)-len(line.lstrip(stripstr)) 107 if firstPass: 108 firstsubnspace=subnspace 109 firstPass=False 110 else: 111 if firstsubnspace!=subnspace: 112 err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n" 113 if len(fileNums)>0: 114 raise Exception(err+"Check indentation at line number: "+str(lineNum)) 115 else: 116 raise Exception(err) 117 118 119 return newTestStr 120 121def _getNewArgs(kvar,val,argStr): 122 """ 123 Given: String containing arguments 124 Return: String without the value in it 125 """ 126 return newTestStr 127 128def _getVarVals(findvar,testDict): 129 """ 130 Given: variable that is either nsize or in args 131 Return: Values to loop over 132 """ 133 vals=None 134 newargs='' 135 if findvar=='nsize': 136 vals=testDict[findvar] 137 else: 138 varlist=[] 139 for varset in re.split('-(?=[a-zA-Z])',testDict['args']): 140 if not varset.strip(): continue 141 if len(re.findall('{{(.*?)}}',varset))>0: 142 # Assuming only one for loop per var specification 143 keyvar=varset.split("{{")[0].strip() 144 if keyvar!=findvar: 145 newargs+="-"+varset+" " 146 continue 147 vals=re.findall('{{(.*?)}}',varset)[0] 148 149 if not vals: raise StandardError("Could not find separate_testvar: "+findvar) 150 return vals,newargs 151 152def genTestsSeparateTestvars(intests,indicts): 153 """ 154 Given: testname, sdict with separate_testvars 155 Return: testnames,sdicts: List of generated tests 156 """ 157 testnames=[]; sdicts=[] 158 for i in range(len(intests)): 159 testname=intests[i]; sdict=indicts[i]; i+=1 160 if sdict.has_key('separate_testvars'): 161 for kvar in sdict['separate_testvars'].split(): 162 kvals,newargs=_getVarVals(kvar,sdict) 163 # No errors means we are good to go 164 for val in kvals.split(): 165 kvardict=sdict.copy() 166 del kvardict['separate_testvars'] 167 gensuffix="_"+kvar+"-"+val 168 newtestnm=testname+gensuffix 169 kvardict['suffix']=sdict['suffix']+gensuffix 170 if kvar=='nsize': 171 kvardict[kvar]=val 172 else: 173 kvardict['args']=newargs+"-"+kvar+" "+val 174 testnames.append(newtestnm) 175 sdicts.append(kvardict) 176 else: 177 testnames.append(testname) 178 sdicts.append(sdict) 179 return testnames,sdicts 180 181def genTestsSubtestSuffix(testnames,sdicts): 182 """ 183 Given: testname, sdict with separate_testvars 184 Return: testnames,sdicts: List of generated tests 185 """ 186 tnms=[]; sdcts=[] 187 for i in range(len(testnames)): 188 testname=testnames[i] 189 rmsubtests=[]; keepSubtests=False 190 if sdicts[i].has_key('subtests'): 191 for stest in sdicts[i]["subtests"]: 192 if sdicts[i][stest].has_key('suffix'): 193 rmsubtests.append(stest) 194 gensuffix="_"+sdicts[i][stest]['suffix'] 195 newtestnm=testname+gensuffix 196 tnms.append(newtestnm) 197 newsdict=sdicts[i].copy() 198 del newsdict['subtests'] 199 # Have to hand update 200 # Append 201 for kup in appendlist: 202 if sdicts[i][stest].has_key(kup): 203 if sdicts[i].has_key(kup): 204 newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup] 205 else: 206 newsdict[kup]=sdicts[i][stest][kup] 207 # Promote 208 for kup in acceptedkeys: 209 if kup in appendlist: continue 210 if sdicts[i][stest].has_key(kup): 211 newsdict[kup]=sdicts[i][stest][kup] 212 # Cleanup 213 for st in sdicts[i]["subtests"]: del newsdict[st] 214 sdcts.append(newsdict) 215 else: 216 keepSubtests=True 217 else: 218 tnms.append(testnames[i]) 219 sdcts.append(sdicts[i]) 220 # Prune the tests to prepare for keeping 221 for rmtest in rmsubtests: 222 sdicts[i]['subtests'].remove(rmtest) 223 del sdicts[i][rmtest] 224 # If we are keeping any tests, then append 225 if keepSubtests: 226 tnms.append(testnames[i]) 227 sdcts.append(sdicts[i]) 228 i+=1 229 return tnms,sdcts 230 231def splitTests(testname,sdict): 232 """ 233 Given: testname and YAML-generated dictionary 234 Return: list of names and dictionaries corresponding to each test 235 given that the YAML language allows for multiple tests 236 """ 237 238 # Order: Parent sep_tv, subtests suffix, subtests sep_tv 239 testnames,sdicts=genTestsSeparateTestvars([testname],[sdict]) 240 testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts) 241 testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts) 242 243 # Because I am altering the list, I do this in passes. Inelegant 244 245 return testnames, sdicts 246 247def parseTest(testStr,srcfile): 248 """ 249 This parses an individual test 250 YAML is hierarchial so should use a state machine in the general case, 251 but in practice we only support two levels of test: 252 """ 253 basename=os.path.basename(srcfile) 254 # Handle the new at the begininng 255 bn=re.sub("new_","",basename) 256 # This is the default 257 testname="run"+os.path.splitext(bn)[0] 258 259 # Tests that have default everything (so empty effectively) 260 if len(testStr)==0: return testname, {} 261 262 striptest=_stripIndent(testStr,srcfile) 263 264 # go through and parse 265 subtestnum=0 266 subdict={} 267 comments=[] 268 indentlevel=0 269 for ln in striptest.split("\n"): 270 line=ln.split('#')[0].rstrip() 271 comment=("" if len(ln.split("#"))==1 else " ".join(ln.split("#")[1:]).strip()) 272 if comment: comments.append(comment) 273 if not line.strip(): continue 274 lsplit=line.split(':') 275 if len(lsplit)==0: raise Exception("Missing : in line: "+line) 276 indentcount=lsplit[0].count(" ") 277 var=lsplit[0].strip() 278 val=lsplit[1].strip() 279 if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from: "+line) 280 # Start by seeing if we are in a subtest 281 if line.startswith(" "): 282 subdict[subtestname][var]=val 283 if not indentlevel: indentlevel=indentcount 284 #if indentlevel!=indentcount: print "Error in indentation:", ln 285 # Determine subtest name and make dict 286 elif var=="test": 287 subtestname="test"+str(subtestnum) 288 subdict[subtestname]={} 289 if not subdict.has_key("subtests"): subdict["subtests"]=[] 290 subdict["subtests"].append(subtestname) 291 subtestnum=subtestnum+1 292 # The rest are easy 293 else: 294 # For convenience, it is sometimes convenient to list twice 295 if subdict.has_key(var): 296 if var in appendlist: 297 subdict[var]+=" "+val 298 else: 299 raise Exception(var+" entered twice: "+line) 300 else: 301 subdict[var]=val 302 if var=="suffix": 303 if len(val)>0: 304 testname=testname+"_"+val 305 306 if len(comments): subdict['comments']="\n".join(comments).lstrip("\n") 307 # A test block can create multiple tests. This does 308 # that logic 309 testnames,subdicts=splitTests(testname,subdict) 310 return testnames,subdicts 311 312def parseTests(testStr,srcfile,fileNums): 313 """ 314 Parse the yaml string describing tests and return 315 a dictionary with the info in the form of: 316 testDict[test][subtest] 317 This is an inelegant parser as we do not wish to 318 introduce a new dependency by bringing in pyyaml. 319 The advantage is that validation can be done as 320 it is parsed (e.g., 'test' is the only top-level node) 321 """ 322 323 testDict={} 324 325 # The first entry should be test: but it might be indented. 326 newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums) 327 328 # Now go through each test. First elem in split is blank 329 for test in newTestStr.split("\ntest:")[1:]: 330 testnames,subdicts=parseTest(test,srcfile) 331 for i in range(len(testnames)): 332 if testDict.has_key(testnames[i]): 333 raise Error("Multiple test names specified: "+testname+" in file: "+srcfile) 334 testDict[testnames[i]]=subdicts[i] 335 336 return testDict 337 338def parseTestFile(srcfile): 339 """ 340 Parse single example files and return dictionary of the form: 341 testDict[srcfile][test][subtest] 342 """ 343 debug=False 344 curdir=os.path.realpath(os.path.curdir) 345 basedir=os.path.dirname(os.path.realpath(srcfile)) 346 basename=os.path.basename(srcfile) 347 os.chdir(basedir) 348 349 testDict={} 350 sh=open(srcfile,"r"); fileStr=sh.read(); sh.close() 351 352 ## Start with doing the tests 353 # 354 fsplit=fileStr.split("/*TEST\n")[1:] 355 if len(fsplit)==0: 356 if debug: print "No test found in: "+srcfile 357 return {} 358 fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1 359 # Allow for multiple "/*TEST" blocks even though it really should be 360 # on 361 srcTests=[] 362 for t in fsplit: srcTests.append(t.split("TEST*/")[0]) 363 testString=" ".join(srcTests) 364 if len(testString.strip())==0: 365 print "No test found in: "+srcfile 366 return {} 367 flen=len(testString.split("\n")) 368 fend=fstart+flen-1 369 fileNums=range(fstart,fend) 370 testDict[basename]=parseTests(testString,srcfile,fileNums) 371 372 ## Check and see if we have build reuqirements 373 # 374 if "/*T\n" in fileStr or "/*T " in fileStr: 375 # The file info is already here and need to append 376 Part1=fileStr.split("T*/")[0] 377 fileInfo=Part1.split("/*T")[1] 378 for bkey in buildkeys: 379 if bkey+":" in fileInfo: 380 testDict[basename][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip() 381 382 os.chdir(curdir) 383 return testDict 384 385def parseTestDir(directory): 386 """ 387 Parse single example files and return dictionary of the form: 388 testDict[srcfile][test][subtest] 389 """ 390 curdir=os.path.realpath(os.path.curdir) 391 basedir=os.path.realpath(directory) 392 os.chdir(basedir) 393 394 tDict={} 395 for test_file in glob.glob("new_ex*.*"): 396 tDict.update(parseTestFile(test_file)) 397 398 os.chdir(curdir) 399 return tDict 400 401def printExParseDict(rDict): 402 """ 403 This is useful for debugging 404 """ 405 indent=" " 406 for sfile in rDict: 407 print sfile 408 sortkeys=rDict[sfile].keys() 409 sortkeys.sort() 410 for runex in sortkeys: 411 print indent+runex 412 if type(rDict[sfile][runex])==types.StringType: 413 print indent*2+rDict[sfile][runex] 414 else: 415 for var in rDict[sfile][runex]: 416 if var.startswith("test"): continue 417 print indent*2+var+": "+str(rDict[sfile][runex][var]) 418 if rDict[sfile][runex].has_key('subtests'): 419 for var in rDict[sfile][runex]['subtests']: 420 print indent*2+var 421 for var2 in rDict[sfile][runex][var]: 422 print indent*3+var2+": "+str(rDict[sfile][runex][var][var2]) 423 print "\n" 424 return 425 426def main(directory='',test_file='',verbosity=0): 427 428 if directory: 429 tDict=parseTestDir(directory) 430 else: 431 tDict=parseTestFile(test_file) 432 if verbosity>0: printExParseDict(tDict) 433 434 return 435 436if __name__ == '__main__': 437 import optparse 438 parser = optparse.OptionParser() 439 parser.add_option('-d', '--directory', dest='directory', 440 default="", help='Directory containing files to parse') 441 parser.add_option('-t', '--test_file', dest='test_file', 442 default="", help='Test file, e.g., ex1.c, to parse') 443 parser.add_option('-v', '--verbosity', dest='verbosity', 444 help='Verbosity of output by level: 1, 2, or 3', default=0) 445 opts, extra_args = parser.parse_args() 446 447 if extra_args: 448 import sys 449 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 450 exit(1) 451 if not opts.test_file and not opts.directory: 452 print "test file or directory is required" 453 parser.print_usage() 454 sys.exit() 455 456 # Need verbosity to be an integer 457 try: 458 verbosity=int(opts.verbosity) 459 except: 460 raise Exception("Error: Verbosity must be integer") 461 462 main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity) 463