1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double __float128 int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 from testparse import parseLoopArgs 107 for key in lkeys: 108 if type(inDict[key])!=types.StringType: continue 109 keystr = str(inDict[key]) 110 akey=('subargs' if key=='args' else key) # what to assign 111 if akey not in inDict: inDict[akey]='' 112 varlist=[] 113 for varset in re.split('-(?=[a-zA-Z])',keystr): 114 if not varset.strip(): continue 115 if '{{' in varset: 116 keyvar,lvars,ftype=parseLoopArgs(varset) 117 if akey not in loopVars: loopVars[akey]={} 118 varlist.append(keyvar) 119 loopVars[akey][keyvar]=[keyvar,lvars] 120 if akey=='nsize': 121 inDict[akey] = '${' + keyvar + '}' 122 lsuffix+=akey+'-'+inDict[akey]+'_' 123 else: 124 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 125 lsuffix+=keyvar+'-${' + keyvar + '}_' 126 else: 127 if key=='args': newargs+=" -"+varset.strip() 128 if len(varlist)>0: loopVars[akey]['varlist']=varlist 129 130 131 # For subtests, args are always substituted in (not top level) 132 if isSubtest: 133 inDict['subargs']+=" "+newargs.strip() 134 inDict['args']='' 135 if 'label_suffix' in inDict: 136 inDict['label_suffix']+=lsuffix.rstrip('_') 137 else: 138 inDict['label_suffix']=lsuffix.rstrip('_') 139 else: 140 if len(loopVars.keys())>0: 141 inDict['args']=newargs.strip() 142 inDict['label_suffix']=lsuffix.rstrip('_') 143 if len(loopVars.keys())>0: 144 return loopVars 145 else: 146 return None 147 148 def getArgLabel(self,testDict): 149 """ 150 In all of the arguments in the test dictionary, create a simple 151 string for searching within the makefile system. For simplicity in 152 search, remove "-", for strings, etc. 153 Also, concatenate the arg commands 154 For now, ignore nsize -- seems hard to search for anyway 155 """ 156 # Collect all of the args associated with a test 157 argStr=("" if 'args' not in testDict else testDict['args']) 158 if 'subtests' in testDict: 159 for stest in testDict["subtests"]: 160 sd=testDict[stest] 161 argStr=argStr+("" if 'args' not in sd else sd['args']) 162 163 # Now go through and cleanup 164 argStr=re.sub('{{(.*?)}}',"",argStr) 165 argStr=re.sub('-'," ",argStr) 166 for digit in string.digits: argStr=re.sub(digit," ",argStr) 167 argStr=re.sub("\.","",argStr) 168 argStr=re.sub(",","",argStr) 169 argStr=re.sub('\+',' ',argStr) 170 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 171 return argStr.strip() 172 173 def addToSources(self,exfile,root,srcDict): 174 """ 175 Put into data structure that allows easy generation of makefile 176 """ 177 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 178 fullfile=os.path.join(root,exfile) 179 relpfile=self.relpath(self.petsc_dir,fullfile) 180 lang=self.getLanguage(exfile) 181 if not lang: return 182 self.sources[pkg][lang]['srcs'].append(relpfile) 183 if 'depends' in srcDict: 184 depSrc=srcDict['depends'] 185 depObj=os.path.splitext(depSrc)[0]+".o" 186 self.sources[pkg][lang][exfile]=depObj 187 188 # In gmakefile, ${TESTDIR} var specifies the object compilation 189 testsdir=self.relpath(self.petsc_dir,root)+"/" 190 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 191 self.objects[pkg].append(objfile) 192 return 193 194 def addToTests(self,test,root,exfile,execname,testDict): 195 """ 196 Put into data structure that allows easy generation of makefile 197 Organized by languages to allow testing of languages 198 """ 199 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 200 #nmtest=self.nameSpace(test,root) 201 rpath=self.relpath(self.petsc_dir,root) 202 nmtest=os.path.join(rpath,test) 203 lang=self.getLanguage(exfile) 204 if not lang: return 205 self.tests[pkg][lang][nmtest]={} 206 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 207 self.tests[pkg][lang][nmtest]['exec']=execname 208 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 209 return 210 211 def getExecname(self,exfile,root): 212 """ 213 Generate bash script using template found next to this file. 214 This file is read in at constructor time to avoid file I/O 215 """ 216 rpath=self.relpath(self.petsc_dir,root) 217 if self.single_ex: 218 execname=rpath.split("/")[1]+"-ex" 219 else: 220 execname=os.path.splitext(exfile)[0] 221 return execname 222 223 def getSubstVars(self,testDict,rpath,testname): 224 """ 225 Create a dictionary with all of the variables that get substituted 226 into the template commands found in example_template.py 227 """ 228 subst={} 229 230 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 231 if 'nsize' not in testDict: testDict['nsize']=1 232 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 233 for ak in testparse.acceptedkeys: 234 if ak=='test': continue 235 subst[ak]=(testDict[ak] if ak in testDict else '') 236 237 # Now do other variables 238 subst['execname']=testDict['execname'] 239 if 'filter' in testDict: 240 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 241 242 # Others 243 subst['subargs']='' # Default. For variables override 244 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 245 subst['label_suffix']='' 246 subst['comments']="\n#".join(subst['comments'].split("\n")) 247 if subst['comments']: subst['comments']="#"+subst['comments'] 248 subst['exec']="../"+subst['execname'] 249 subst['testroot']=self.testroot_dir 250 subst['testname']=testname 251 dp = self.conf.get('DATAFILESPATH','') 252 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 253 254 # This is used to label some matrices 255 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 256 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 257 258 # These can have for loops and are treated separately later 259 subst['nsize']=str(subst['nsize']) 260 261 #Conf vars 262 if self.petsc_arch.find('valgrind')>=0: 263 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 264 else: 265 subst['mpiexec']=self.conf['MPIEXEC'] 266 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 267 subst['petsc_arch']=self.petsc_arch 268 subst['diff']=self.conf['DIFF'] 269 subst['rm']=self.conf['RM'] 270 subst['grep']=self.conf['GREP'] 271 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 272 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 273 274 # Output file is special because of subtests override 275 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 276 if not "_" in defroot: defroot=defroot+"_1" 277 subst['defroot']=defroot 278 subst['label']=self.nameSpace(defroot,subst['srcdir']) 279 subst['redirect_file']=defroot+".tmp" 280 if 'output_file' not in testDict: 281 subst['output_file']="output/"+defroot+".out" 282 # Add in the full path here. 283 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 284 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 285 if not subst['TODO']: 286 print "Warning: "+subst['output_file']+" not found." 287 # Worry about alt files here -- see 288 # src/snes/examples/tutorials/output/ex22*.out 289 altlist=[subst['output_file']] 290 for i in range(1,3): 291 altroot=defroot+"_alt" 292 if i==2: altroot=altroot+"_2" 293 af="output/"+altroot+".out" 294 srcaf=os.path.join(subst['srcdir'],af) 295 fullaf=os.path.join(self.petsc_dir,srcaf) 296 if os.path.isfile(fullaf): altlist.append(srcaf) 297 if len(altlist)>1: subst['altfiles']=altlist 298 #if len(altlist)>1: print "Found alt files: ",altlist 299 300 return subst 301 302 def getCmds(self,subst,i): 303 """ 304 Generate bash script using template found next to this file. 305 This file is read in at constructor time to avoid file I/O 306 """ 307 nindnt=i # the start and has to be consistent with below 308 cmdindnt=self.indent*nindnt 309 cmdLines="" 310 311 # MPI is the default -- but we have a few odd commands 312 if not subst['command']: 313 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 314 else: 315 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 316 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 317 318 cmdLines+=cmdindnt+'if test $res = 0; then\n' 319 diffindnt=self.indent*(nindnt+1) 320 if not subst['filter_output']: 321 if 'altfiles' not in subst: 322 cmd=diffindnt+self._substVars(subst,example_template.difftest) 323 else: 324 # Have to do it by hand a bit because of variable number of alt files 325 rf=subst['redirect_file'] 326 cmd=diffindnt+example_template.difftest.split('@')[0] 327 for i in range(len(subst['altfiles'])): 328 af=subst['altfiles'][i] 329 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 330 if i!=len(subst['altfiles'])-1: 331 cmd+=' || ${diff_exe} ' 332 else: 333 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 334 cmd+=subst['label_suffix']+' ""' # Quotes are painful 335 else: 336 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 337 cmdLines+=cmd+"\n" 338 cmdLines+=cmdindnt+'else\n' 339 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 340 cmdLines+=cmdindnt+'fi\n' 341 return cmdLines 342 343 def _substVars(self,subst,origStr): 344 """ 345 Substitute variables 346 """ 347 Str=origStr 348 for subkey in subst: 349 if type(subst[subkey])!=types.StringType: continue 350 patt="@"+subkey.upper()+"@" 351 Str=re.sub(patt,subst[subkey],Str) 352 return Str 353 354 def _writeTodoSkip(self,fh,tors,reasons,footer): 355 """ 356 Write out the TODO and SKIP lines in the file 357 The TODO or SKIP variable, tors, should be lower case 358 """ 359 TORS=tors.upper() 360 template=eval("example_template."+tors+"line") 361 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 362 tab = '' 363 if reasons: 364 fh.write('if ! $force; then\n') 365 tab = tab + ' ' 366 if reasons == ["Requires DATAFILESPATH"]: 367 # The only reason not to run is DATAFILESPATH, which we check at run-time 368 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 369 tab = tab + ' ' 370 if reasons: 371 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 372 fh.write(tab+footer+"\n") 373 fh.write(tab+"exit\n") 374 if reasons == ["Requires DATAFILESPATH"]: 375 fh.write(' fi\n') 376 if reasons: 377 fh.write('fi\n') 378 fh.write('\n\n') 379 return 380 381 def getLoopVarsHead(self,loopVars,i): 382 """ 383 Generate a nicely indented string with the format loops 384 Here is what the data structure looks like 385 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 386 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 387 loopVars['subargs']['pc_type']=["j","cholesky sor"] 388 """ 389 outstr=''; indnt=self.indent 390 for key in loopVars: 391 for var in loopVars[key]['varlist']: 392 varval=loopVars[key][var] 393 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 394 i = i + 1 395 return (outstr,i) 396 397 def getLoopVarsFoot(self,loopVars,i): 398 outstr=''; indnt=self.indent 399 for key in loopVars: 400 for var in loopVars[key]['varlist']: 401 i = i - 1 402 outstr += indnt * i + "done\n" 403 return (outstr,i) 404 405 def genRunScript(self,testname,root,isRun,srcDict): 406 """ 407 Generate bash script using template found next to this file. 408 This file is read in at constructor time to avoid file I/O 409 """ 410 # runscript_dir directory has to be consistent with gmakefile 411 testDict=srcDict[testname] 412 rpath=self.relpath(self.petsc_dir,root) 413 runscript_dir=os.path.join(self.testroot_dir,rpath) 414 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 415 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 416 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 417 418 # Get variables to go into shell scripts. last time testDict used 419 subst=self.getSubstVars(testDict,rpath,testname) 420 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 421 #if '33_' in testname: print subst['subargs'] 422 423 #Handle runfiles 424 for lfile in subst.get('localrunfiles','').split(): 425 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 426 shutil.copy(fullfile,runscript_dir) 427 # Check subtests for local runfiles 428 for stest in subst.get("subtests",[]): 429 for lfile in testDict[stest].get('localrunfiles','').split(): 430 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 431 shutil.copy(fullfile,self.runscript_dir) 432 433 # Now substitute the key variables into the header and footer 434 header=self._substVars(subst,example_template.header) 435 # The header is done twice to enable @...@ in header 436 header=self._substVars(subst,header) 437 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 438 439 # Start writing the file 440 fh.write(header+"\n") 441 442 # If there is a TODO or a SKIP then we do it before writing out the 443 # rest of the command (which is useful for working on the test) 444 # SKIP and TODO can be for the source file or for the runs 445 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 446 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 447 448 j=0 # for indentation 449 450 if loopVars: 451 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 452 if (loopHead): fh.write(loopHead+"\n") 453 454 # Subtests are special 455 if 'subtests' in testDict: 456 substP=subst # Subtests can inherit args but be careful 457 for stest in testDict["subtests"]: 458 subst=substP.copy() 459 subst.update(testDict[stest]) 460 subst['nsize']=str(subst['nsize']) 461 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 462 #if '10_9' in testname: print sLoopVars 463 if sLoopVars: 464 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 465 fh.write(sLoopHead+"\n") 466 fh.write(self.getCmds(subst,j)+"\n") 467 if sLoopVars: 468 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 469 fh.write(sLoopFoot+"\n") 470 else: 471 fh.write(self.getCmds(subst,j)+"\n") 472 473 if loopVars: 474 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 475 fh.write(loopFoot+"\n") 476 477 fh.write(footer+"\n") 478 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 479 #if '10_9' in testname: sys.exit() 480 return 481 482 def genScriptsAndInfo(self,exfile,root,srcDict): 483 """ 484 Generate scripts from the source file, determine if built, etc. 485 For every test in the exfile with info in the srcDict: 486 1. Determine if it needs to be run for this arch 487 2. Generate the script 488 3. Generate the data needed to write out the makefile in a 489 convenient way 490 All tests are *always* run, but some may be SKIP'd per the TAP standard 491 """ 492 debug=False 493 execname=self.getExecname(exfile,root) 494 isBuilt=self._isBuilt(exfile,srcDict) 495 for test in srcDict: 496 if test in self.buildkeys: continue 497 if debug: print self.nameSpace(exfile,root), test 498 srcDict[test]['execname']=execname # Convenience in generating scripts 499 isRun=self._isRun(srcDict[test]) 500 self.genRunScript(test,root,isRun,srcDict) 501 srcDict[test]['isrun']=isRun 502 self.addToTests(test,root,exfile,execname,srcDict[test]) 503 504 # This adds to datastructure for building deps 505 if isBuilt: self.addToSources(exfile,root,srcDict) 506 return 507 508 def _isBuilt(self,exfile,srcDict): 509 """ 510 Determine if this file should be built. 511 """ 512 # Get the language based on file extension 513 srcDict['SKIP'] = [] 514 lang=self.getLanguage(exfile) 515 if (lang=="F" or lang=="F90"): 516 if not self.have_fortran: 517 srcDict["SKIP"].append("Fortran required for this test") 518 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 519 srcDict["SKIP"].append("Fortran f90freeform required for this test") 520 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 521 srcDict["SKIP"].append("CUDA required for this test") 522 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 523 srcDict["SKIP"].append("C++ required for this test") 524 525 # Deprecated source files 526 if srcDict.get("TODO"): 527 return False 528 529 # isRun can work with srcDict to handle the requires 530 if "requires" in srcDict: 531 if len(srcDict["requires"])>0: 532 return self._isRun(srcDict) 533 534 return srcDict['SKIP'] == [] 535 536 537 def _isRun(self,testDict): 538 """ 539 Based on the requirements listed in the src file and the petscconf.h 540 info, determine whether this test should be run or not. 541 """ 542 indent=" " 543 debug=False 544 545 if 'SKIP' not in testDict: 546 testDict['SKIP'] = [] 547 # MPI requirements 548 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 549 if debug: print indent+"Cannot run parallel tests" 550 testDict['SKIP'].append("Parallel test with serial build") 551 552 # The requirements for the test are the sum of all the run subtests 553 if 'subtests' in testDict: 554 if 'requires' not in testDict: testDict['requires']="" 555 for stest in testDict['subtests']: 556 if 'requires' in testDict[stest]: 557 testDict['requires']+=" "+testDict[stest]['requires'] 558 559 560 # Now go through all requirements 561 if 'requires' in testDict: 562 for requirement in testDict['requires'].split(): 563 requirement=requirement.strip() 564 if not requirement: continue 565 if debug: print indent+"Requirement: ", requirement 566 isNull=False 567 if requirement.startswith("!"): 568 requirement=requirement[1:]; isNull=True 569 # Precision requirement for reals 570 if requirement in self.precision_types: 571 if self.conf['PETSC_PRECISION']==requirement: 572 if isNull: 573 testDict['SKIP'].append("not "+requirement+" required") 574 continue 575 continue # Success 576 elif not isNull: 577 testDict['SKIP'].append(requirement+" required") 578 continue 579 # Precision requirement for ints 580 if requirement in self.integer_types: 581 if requirement=="int32": 582 if self.conf['PETSC_SIZEOF_INT']==4: 583 if isNull: 584 testDict['SKIP'].append("not int32 required") 585 continue 586 continue # Success 587 elif not isNull: 588 testDict['SKIP'].append("int32 required") 589 continue 590 if requirement=="int64": 591 if self.conf['PETSC_SIZEOF_INT']==8: 592 if isNull: 593 testDict['SKIP'].append("NOT int64 required") 594 continue 595 continue # Success 596 elif not isNull: 597 testDict['SKIP'].append("int64 required") 598 continue 599 # Datafilespath 600 if requirement=="datafilespath" and not isNull: 601 testDict['SKIP'].append("Requires DATAFILESPATH") 602 continue 603 # Defines -- not sure I have comments matching 604 if "define(" in requirement.lower(): 605 reqdef=requirement.split("(")[1].split(")")[0] 606 if reqdef in self.conf: 607 if isNull: 608 testDict['SKIP'].append("Null requirement not met: "+requirement) 609 continue 610 continue # Success 611 elif not isNull: 612 testDict['SKIP'].append("Required: "+requirement) 613 continue 614 615 # Rest should be packages that we can just get from conf 616 if requirement == "complex": 617 petscconfvar="PETSC_USE_COMPLEX" 618 else: 619 petscconfvar="PETSC_HAVE_"+requirement.upper() 620 if self.conf.get(petscconfvar): 621 if isNull: 622 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 623 continue 624 continue # Success 625 elif not isNull: 626 if debug: print "requirement not found: ", requirement 627 testDict['SKIP'].append(petscconfvar+" requirement not met") 628 continue 629 630 return testDict['SKIP'] == [] 631 632 def genPetscTests_summarize(self,dataDict): 633 """ 634 Required method to state what happened 635 """ 636 if not self.summarize: return 637 indent=" " 638 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 639 fh=open(fhname,"w") 640 #print "See ", fhname 641 for root in dataDict: 642 relroot=self.relpath(self.petsc_dir,root) 643 pkg=relroot.split("/")[1] 644 fh.write(relroot+"\n") 645 allSrcs=[] 646 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 647 for exfile in dataDict[root]: 648 # Basic information 649 fullfile=os.path.join(root,exfile) 650 rfile=self.relpath(self.petsc_dir,fullfile) 651 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 652 fh.write(indent+exfile+indent*4+builtStatus+"\n") 653 654 for test in dataDict[root][exfile]: 655 if test in self.buildkeys: continue 656 line=indent*2+test 657 fh.write(line+"\n") 658 # Looks nice to have the keys in order 659 #for key in dataDict[root][exfile][test]: 660 for key in "isrun abstracted nsize args requires script".split(): 661 if key not in dataDict[root][exfile][test]: continue 662 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 663 fh.write(line+"\n") 664 fh.write("\n") 665 fh.write("\n") 666 fh.write("\n") 667 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 668 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 669 fh.close() 670 return 671 672 def genPetscTests(self,root,dirs,files,dataDict): 673 """ 674 Go through and parse the source files in the directory to generate 675 the examples based on the metadata contained in the source files 676 """ 677 debug=False 678 # Use examplesAnalyze to get what the makefles think are sources 679 #self.examplesAnalyze(root,dirs,files,anlzDict) 680 681 dataDict[root]={} 682 683 for exfile in files: 684 #TST: Until we replace files, still leaving the orginals as is 685 #if not exfile.startswith("new_"+"ex"): continue 686 #if not exfile.startswith("ex"): continue 687 688 # Convenience 689 fullex=os.path.join(root,exfile) 690 relpfile=self.relpath(self.petsc_dir,fullex) 691 if debug: print relpfile 692 dataDict[root].update(testparse.parseTestFile(fullex,0)) 693 # Need to check and make sure tests are in the file 694 # if verbosity>=1: print relpfile 695 if exfile in dataDict[root]: 696 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 697 698 return 699 700 def walktree(self,top,action="printFiles"): 701 """ 702 Walk a directory tree, starting from 'top' 703 """ 704 #print "action", action 705 # Goal of action is to fill this dictionary 706 dataDict={} 707 for root, dirs, files in os.walk(top, topdown=False): 708 if not "examples" in root: continue 709 if not os.path.isfile(os.path.join(root,"makefile")): continue 710 bname=os.path.basename(root.rstrip("/")) 711 if bname=="tests" or bname=="tutorials": 712 eval("self."+action+"(root,dirs,files,dataDict)") 713 if type(top) != types.StringType: 714 raise TypeError("top must be a string") 715 # Now summarize this dictionary 716 eval("self."+action+"_summarize(dataDict)") 717 return dataDict 718 719 def gen_gnumake(self, fd): 720 """ 721 Overwrite of the method in the base PETSc class 722 """ 723 def write(stem, srcs): 724 for lang in LANGS: 725 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 726 for pkg in PKGS: 727 srcs = self.gen_pkg(pkg) 728 write('testsrcs-' + pkg, srcs) 729 return self.gendeps 730 731 def gen_pkg(self, pkg): 732 """ 733 Overwrite of the method in the base PETSc class 734 """ 735 return self.sources[pkg] 736 737 def write_gnumake(self,dataDict): 738 """ 739 Write out something similar to files from gmakegen.py 740 741 Test depends on script which also depends on source 742 file, but since I don't have a good way generating 743 acting on a single file (oops) just depend on 744 executable which in turn will depend on src file 745 """ 746 # Different options for how to set up the targets 747 compileExecsFirst=False 748 749 # Open file 750 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 751 fd = open(arch_files, 'w') 752 753 # Write out the sources 754 gendeps = self.gen_gnumake(fd) 755 756 # Write out the tests and execname targets 757 fd.write("\n#Tests and executables\n") # Delimiter 758 759 for pkg in PKGS: 760 # These grab the ones that are built 761 for lang in LANGS: 762 testdeps=[] 763 for ftest in self.tests[pkg][lang]: 764 test=os.path.basename(ftest) 765 basedir=os.path.dirname(ftest) 766 testdeps.append(self.nameSpace(test,basedir)) 767 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 768 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 769 770 # test targets 771 for ftest in self.tests[pkg][lang]: 772 test=os.path.basename(ftest) 773 basedir=os.path.dirname(ftest) 774 testdir="${TESTDIR}/"+basedir+"/" 775 nmtest=self.nameSpace(test,basedir) 776 rundir=os.path.join(testdir,test) 777 #print test, nmtest 778 script=test+".sh" 779 780 # Deps 781 exfile=self.tests[pkg][lang][ftest]['exfile'] 782 fullex=os.path.join(self.petsc_dir,exfile) 783 localexec=self.tests[pkg][lang][ftest]['exec'] 784 execname=os.path.join(testdir,localexec) 785 fullscript=os.path.join(testdir,script) 786 tmpfile=os.path.join(testdir,test,test+".tmp") 787 788 # *.counts depends on the script and either executable (will 789 # be run) or the example source file (SKIP or TODO) 790 fd.write('%s.counts : %s %s\n' 791 % (os.path.join('$(TESTDIR)/counts', nmtest), 792 fullscript, 793 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 794 # Now write the args: 795 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 796 797 fd.close() 798 return 799 800 def writeHarness(self,output,dataDict): 801 """ 802 This is set up to write out multiple harness even if only gnumake 803 is supported now 804 """ 805 eval("self.write_"+output+"(dataDict)") 806 return 807 808def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 809 if output is None: 810 output = 'gnumake' 811 812 813 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 814 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 815 pEx.writeHarness(output,dataDict) 816 817if __name__ == '__main__': 818 import optparse 819 parser = optparse.OptionParser() 820 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 821 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 822 parser.add_option('--output', help='Location to write output file', default=None) 823 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 824 opts, extra_args = parser.parse_args() 825 if extra_args: 826 import sys 827 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 828 exit(1) 829 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 830