1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False): 25 super(generateExamples, self).__init__(petsc_dir=None, petsc_arch=None, verbose=False) 26 27 self.single_ex=single_ex 28 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 29 self.ptNaming=True 30 # Whether to write out a useful debugging 31 #if verbose: self.summarize=True 32 self.summarize=True 33 34 # For help in setting the requirements 35 self.precision_types="single double __float128 int32".split() 36 self.integer_types="int32 int64".split() 37 self.languages="fortran cuda cxx".split() # Always requires C so do not list 38 39 # Things that are not test 40 self.buildkeys=testparse.buildkeys 41 42 # Adding a dictionary for storing sources, objects, and tests 43 # to make building the dependency tree easier 44 self.sources={} 45 self.objects={} 46 self.tests={} 47 for pkg in PKGS: 48 self.sources[pkg]={} 49 self.objects[pkg]=[] 50 self.tests[pkg]={} 51 for lang in LANGS: 52 self.sources[pkg][lang]={} 53 self.sources[pkg][lang]['srcs']=[] 54 self.tests[pkg][lang]={} 55 56 # Do some initialization 57 self.testroot_dir=os.path.join(self.arch_dir,"tests") 58 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 59 60 self.indent=" " 61 return 62 63 def nameSpace(self,srcfile,srcdir): 64 """ 65 Because the scripts have a non-unique naming, the pretty-printing 66 needs to convey the srcdir and srcfile. There are two ways of doing this. 67 """ 68 if self.ptNaming: 69 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 70 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 71 nameString=prefix+srcfile 72 else: 73 #nameString=srcdir+": "+srcfile 74 nameString=srcfile 75 return nameString 76 77 def getLanguage(self,srcfile): 78 """ 79 Based on the source, determine associated language as found in gmakegen.LANGS 80 Can we just return srcext[1:\] now? 81 """ 82 langReq=None 83 srcext=os.path.splitext(srcfile)[-1] 84 if srcext in ".F90".split(): langReq="F90" 85 if srcext in ".F".split(): langReq="F" 86 if srcext in ".cxx".split(): langReq="cxx" 87 if srcext == ".cu": langReq="cu" 88 if srcext == ".c": langReq="c" 89 #if not langReq: print "ERROR: ", srcext, srcfile 90 return langReq 91 92 def _getLoopVars(self,inDict,testname, isSubtest=False): 93 """ 94 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 95 Return: 96 inDict['args']: -ksp_monitor 97 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 98 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 99 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 100 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 101 subst should be passed in instead of inDict 102 """ 103 loopVars={}; newargs="" 104 lkeys=inDict.keys() 105 lsuffix='_' 106 from testparse import parseLoopArgs 107 for key in lkeys: 108 if type(inDict[key])!=types.StringType: continue 109 keystr = str(inDict[key]) 110 akey=('subargs' if key=='args' else key) # what to assign 111 if akey not in inDict: inDict[akey]='' 112 varlist=[] 113 for varset in re.split('-(?=[a-zA-Z])',keystr): 114 if not varset.strip(): continue 115 if '{{' in varset: 116 keyvar,lvars,ftype=parseLoopArgs(varset) 117 if akey not in loopVars: loopVars[akey]={} 118 varlist.append(keyvar) 119 loopVars[akey][keyvar]=[keyvar,lvars] 120 if akey=='nsize': 121 inDict[akey] = '${' + keyvar + '}' 122 lsuffix+=akey+'-'+inDict[akey]+'_' 123 else: 124 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 125 lsuffix+=keyvar+'-${' + keyvar + '}_' 126 else: 127 if key=='args': newargs+=" -"+varset.strip() 128 if len(varlist)>0: loopVars[akey]['varlist']=varlist 129 130 131 # For subtests, args are always substituted in (not top level) 132 if isSubtest: 133 inDict['subargs']+=" "+newargs.strip() 134 inDict['args']='' 135 if 'label_suffix' in inDict: 136 inDict['label_suffix']+=lsuffix.rstrip('_') 137 else: 138 inDict['label_suffix']=lsuffix.rstrip('_') 139 else: 140 if len(loopVars.keys())>0: 141 inDict['args']=newargs.strip() 142 inDict['label_suffix']=lsuffix.rstrip('_') 143 if len(loopVars.keys())>0: 144 return loopVars 145 else: 146 return None 147 148 def getArgLabel(self,testDict): 149 """ 150 In all of the arguments in the test dictionary, create a simple 151 string for searching within the makefile system. For simplicity in 152 search, remove "-", for strings, etc. 153 Also, concatenate the arg commands 154 For now, ignore nsize -- seems hard to search for anyway 155 """ 156 # Collect all of the args associated with a test 157 argStr=("" if 'args' not in testDict else testDict['args']) 158 if 'subtests' in testDict: 159 for stest in testDict["subtests"]: 160 sd=testDict[stest] 161 argStr=argStr+("" if 'args' not in sd else sd['args']) 162 163 # Now go through and cleanup 164 argStr=re.sub('{{(.*?)}}',"",argStr) 165 argStr=re.sub('-'," ",argStr) 166 for digit in string.digits: argStr=re.sub(digit," ",argStr) 167 argStr=re.sub("\.","",argStr) 168 argStr=re.sub(",","",argStr) 169 argStr=re.sub('\+',' ',argStr) 170 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 171 return argStr.strip() 172 173 def addToSources(self,exfile,root,srcDict): 174 """ 175 Put into data structure that allows easy generation of makefile 176 """ 177 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 178 fullfile=os.path.join(root,exfile) 179 relpfile=self.relpath(self.petsc_dir,fullfile) 180 lang=self.getLanguage(exfile) 181 if not lang: return 182 self.sources[pkg][lang]['srcs'].append(relpfile) 183 if 'depends' in srcDict: 184 depSrc=srcDict['depends'] 185 depObj=os.path.splitext(depSrc)[0]+".o" 186 self.sources[pkg][lang][exfile]=depObj 187 188 # In gmakefile, ${TESTDIR} var specifies the object compilation 189 testsdir=self.relpath(self.petsc_dir,root)+"/" 190 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 191 self.objects[pkg].append(objfile) 192 return 193 194 def addToTests(self,test,root,exfile,execname,testDict): 195 """ 196 Put into data structure that allows easy generation of makefile 197 Organized by languages to allow testing of languages 198 """ 199 pkg=self.relpath(self.petsc_dir,root).split("/")[1] 200 #nmtest=self.nameSpace(test,root) 201 rpath=self.relpath(self.petsc_dir,root) 202 nmtest=os.path.join(rpath,test) 203 lang=self.getLanguage(exfile) 204 if not lang: return 205 self.tests[pkg][lang][nmtest]={} 206 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 207 self.tests[pkg][lang][nmtest]['exec']=execname 208 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 209 return 210 211 def getExecname(self,exfile,root): 212 """ 213 Generate bash script using template found next to this file. 214 This file is read in at constructor time to avoid file I/O 215 """ 216 rpath=self.relpath(self.petsc_dir,root) 217 if self.single_ex: 218 execname=rpath.split("/")[1]+"-ex" 219 else: 220 execname=os.path.splitext(exfile)[0] 221 return execname 222 223 def getSubstVars(self,testDict,rpath,testname): 224 """ 225 Create a dictionary with all of the variables that get substituted 226 into the template commands found in example_template.py 227 """ 228 subst={} 229 230 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 231 if 'nsize' not in testDict: testDict['nsize']=1 232 for ak in testparse.acceptedkeys: 233 if ak=='test': continue 234 subst[ak]=(testDict[ak] if ak in testDict else '') 235 236 # Now do other variables 237 subst['execname']=testDict['execname'] 238 if 'filter' in testDict: 239 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 240 241 # Others 242 subst['subargs']='' # Default. For variables override 243 subst['srcdir']=os.path.join(self.petsc_dir,rpath) 244 subst['label_suffix']='' 245 subst['comments']="\n#".join(subst['comments'].split("\n")) 246 if subst['comments']: subst['comments']="#"+subst['comments'] 247 subst['exec']="../"+subst['execname'] 248 subst['testroot']=self.testroot_dir 249 subst['testname']=testname 250 dp = self.conf.get('DATAFILESPATH','') 251 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 252 253 # This is used to label some matrices 254 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 255 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 256 257 # These can have for loops and are treated separately later 258 subst['nsize']=str(subst['nsize']) 259 260 #Conf vars 261 if self.petsc_arch.find('valgrind')>=0: 262 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 263 else: 264 subst['mpiexec']=self.conf['MPIEXEC'] 265 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 266 subst['diff']=self.conf['DIFF'] 267 subst['rm']=self.conf['RM'] 268 subst['grep']=self.conf['GREP'] 269 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 270 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 271 272 # Output file is special because of subtests override 273 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 274 if not "_" in defroot: defroot=defroot+"_1" 275 subst['defroot']=defroot 276 subst['label']=self.nameSpace(defroot,subst['srcdir']) 277 subst['redirect_file']=defroot+".tmp" 278 if 'output_file' not in testDict: 279 subst['output_file']="output/"+defroot+".out" 280 # Add in the full path here. 281 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 282 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 283 if not subst['TODO']: 284 print "Warning: "+subst['output_file']+" not found." 285 # Worry about alt files here -- see 286 # src/snes/examples/tutorials/output/ex22*.out 287 altlist=[subst['output_file']] 288 for i in range(1,3): 289 altroot=defroot+"_alt" 290 if i==2: altroot=altroot+"_2" 291 af="output/"+altroot+".out" 292 srcaf=os.path.join(subst['srcdir'],af) 293 fullaf=os.path.join(self.petsc_dir,srcaf) 294 if os.path.isfile(fullaf): altlist.append(srcaf) 295 if len(altlist)>1: subst['altfiles']=altlist 296 #if len(altlist)>1: print "Found alt files: ",altlist 297 298 return subst 299 300 def getCmds(self,subst,i): 301 """ 302 Generate bash script using template found next to this file. 303 This file is read in at constructor time to avoid file I/O 304 """ 305 indnt=self.indent 306 nindnt=i # the start and has to be consistent with below 307 cmdLines="" 308 309 # MPI is the default -- but we have a few odd commands 310 if not subst['command']: 311 cmd=indnt*nindnt+self._substVars(subst,example_template.mpitest) 312 else: 313 cmd=indnt*nindnt+self._substVars(subst,example_template.commandtest) 314 cmdLines+=cmd+"\n\n" 315 316 if not subst['filter_output']: 317 if 'altfiles' not in subst: 318 cmd=indnt*nindnt+self._substVars(subst,example_template.difftest) 319 else: 320 # Have to do it by hand a bit because of variable number of alt files 321 rf=subst['redirect_file'] 322 cmd=indnt*nindnt+example_template.difftest.split('@')[0] 323 for i in range(len(subst['altfiles'])): 324 af=subst['altfiles'][i] 325 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 326 if i!=len(subst['altfiles'])-1: 327 cmd+=' || ${diff_exe} ' 328 else: 329 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 330 cmd+=subst['label_suffix']+' ""' # Quotes are painful 331 else: 332 cmd=indnt*nindnt+self._substVars(subst,example_template.filterdifftest) 333 cmdLines+=cmd+"\n" 334 return cmdLines 335 336 def _substVars(self,subst,origStr): 337 """ 338 Substitute variables 339 """ 340 Str=origStr 341 for subkey in subst: 342 if type(subst[subkey])!=types.StringType: continue 343 patt="@"+subkey.upper()+"@" 344 Str=re.sub(patt,subst[subkey],Str) 345 return Str 346 347 def _writeTodoSkip(self,fh,tors,reasons,footer): 348 """ 349 Write out the TODO and SKIP lines in the file 350 The TODO or SKIP variable, tors, should be lower case 351 """ 352 TORS=tors.upper() 353 template=eval("example_template."+tors+"line") 354 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 355 tab = '' 356 if reasons == ["Requires DATAFILESPATH"]: 357 # The only reason not to run is DATAFILESPATH, which we check at run-time 358 tab = ' ' 359 fh.write('if test -z "${DATAFILESPATH}"; then\n') 360 if reasons: 361 fh.write(tab+tsStr+"\ntotal=1; "+tors+"=1\n") 362 fh.write(tab+footer+"\n") 363 fh.write(tab+"exit\n") 364 if reasons == ["Requires DATAFILESPATH"]: 365 fh.write('fi\n') 366 fh.write('\n\n') 367 return 368 369 def getLoopVarsHead(self,loopVars,i): 370 """ 371 Generate a nicely indented string with the format loops 372 Here is what the data structure looks like 373 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 374 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 375 loopVars['subargs']['pc_type']=["j","cholesky sor"] 376 """ 377 outstr=''; indnt=self.indent 378 for key in loopVars: 379 for var in loopVars[key]['varlist']: 380 varval=loopVars[key][var] 381 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 382 i = i + 1 383 return (outstr,i) 384 385 def getLoopVarsFoot(self,loopVars,i): 386 outstr=''; indnt=self.indent 387 for key in loopVars: 388 for var in loopVars[key]['varlist']: 389 i = i - 1 390 outstr += indnt * i + "done\n" 391 return (outstr,i) 392 393 def genRunScript(self,testname,root,isRun,srcDict): 394 """ 395 Generate bash script using template found next to this file. 396 This file is read in at constructor time to avoid file I/O 397 """ 398 # runscript_dir directory has to be consistent with gmakefile 399 testDict=srcDict[testname] 400 rpath=self.relpath(self.petsc_dir,root) 401 runscript_dir=os.path.join(self.testroot_dir,rpath) 402 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 403 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 404 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 405 406 # Get variables to go into shell scripts. last time testDict used 407 subst=self.getSubstVars(testDict,rpath,testname) 408 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 409 #if '33_' in testname: print subst['subargs'] 410 411 #Handle runfiles 412 for lfile in subst.get('localrunfiles','').split(): 413 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 414 shutil.copy(fullfile,runscript_dir) 415 # Check subtests for local runfiles 416 for stest in subst.get("subtests",[]): 417 for lfile in testDict[stest].get('localrunfiles','').split(): 418 fullfile=os.path.join(self.petsc_dir,rpath,lfile) 419 shutil.copy(fullfile,self.runscript_dir) 420 421 # Now substitute the key variables into the header and footer 422 header=self._substVars(subst,example_template.header) 423 # The header is done twice to enable @...@ in header 424 header=self._substVars(subst,header) 425 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 426 427 # Start writing the file 428 fh.write(header+"\n") 429 430 # If there is a TODO or a SKIP then we do it before writing out the 431 # rest of the command (which is useful for working on the test) 432 # SKIP and TODO can be for the source file or for the runs 433 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 434 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 435 436 j=0 # for indentation 437 438 if loopVars: 439 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 440 if (loopHead): fh.write(loopHead+"\n") 441 442 # Subtests are special 443 if 'subtests' in testDict: 444 substP=subst # Subtests can inherit args but be careful 445 for stest in testDict["subtests"]: 446 subst=substP.copy() 447 subst.update(testDict[stest]) 448 subst['nsize']=str(subst['nsize']) 449 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 450 #if '10_9' in testname: print sLoopVars 451 if sLoopVars: 452 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 453 fh.write(sLoopHead+"\n") 454 fh.write(self.getCmds(subst,j)+"\n") 455 if sLoopVars: 456 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 457 fh.write(sLoopFoot+"\n") 458 else: 459 fh.write(self.getCmds(subst,j)+"\n") 460 461 if loopVars: 462 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 463 fh.write(loopFoot+"\n") 464 465 fh.write(footer+"\n") 466 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 467 #if '10_9' in testname: sys.exit() 468 return 469 470 def genScriptsAndInfo(self,exfile,root,srcDict): 471 """ 472 Generate scripts from the source file, determine if built, etc. 473 For every test in the exfile with info in the srcDict: 474 1. Determine if it needs to be run for this arch 475 2. Generate the script 476 3. Generate the data needed to write out the makefile in a 477 convenient way 478 All tests are *always* run, but some may be SKIP'd per the TAP standard 479 """ 480 debug=False 481 fileIsTested=False 482 execname=self.getExecname(exfile,root) 483 isBuilt=self._isBuilt(exfile,srcDict) 484 for test in srcDict: 485 if test in self.buildkeys: continue 486 if debug: print self.nameSpace(exfile,root), test 487 srcDict[test]['execname']=execname # Convenience in generating scripts 488 isRun=self._isRun(srcDict[test]) 489 self.genRunScript(test,root,isRun,srcDict) 490 srcDict[test]['isrun']=isRun 491 if isRun: fileIsTested=True 492 self.addToTests(test,root,exfile,execname,srcDict[test]) 493 494 # This adds to datastructure for building deps 495 if fileIsTested and isBuilt: self.addToSources(exfile,root,srcDict) 496 #print self.nameSpace(exfile,root), fileIsTested 497 return 498 499 def _isBuilt(self,exfile,srcDict): 500 """ 501 Determine if this file should be built. 502 """ 503 # Get the language based on file extension 504 srcDict['SKIP'] = [] 505 lang=self.getLanguage(exfile) 506 if (lang=="F" or lang=="F90") and not self.have_fortran: 507 srcDict["SKIP"].append("Fortran required for this test") 508 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 509 srcDict["SKIP"].append("CUDA required for this test") 510 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 511 srcDict["SKIP"].append("C++ required for this test") 512 513 # Deprecated source files 514 if srcDict.get("TODO"): 515 return False 516 517 # isRun can work with srcDict to handle the requires 518 if "requires" in srcDict: 519 if len(srcDict["requires"])>0: 520 return self._isRun(srcDict) 521 522 return srcDict['SKIP'] == [] 523 524 525 def _isRun(self,testDict): 526 """ 527 Based on the requirements listed in the src file and the petscconf.h 528 info, determine whether this test should be run or not. 529 """ 530 indent=" " 531 debug=False 532 533 if 'SKIP' not in testDict: 534 testDict['SKIP'] = [] 535 # MPI requirements 536 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 537 if debug: print indent+"Cannot run parallel tests" 538 testDict['SKIP'].append("Parallel test with serial build") 539 540 # The requirements for the test are the sum of all the run subtests 541 if 'subtests' in testDict: 542 if 'requires' not in testDict: testDict['requires']="" 543 for stest in testDict['subtests']: 544 if 'requires' in testDict[stest]: 545 testDict['requires']+=" "+testDict[stest]['requires'] 546 547 548 # Now go through all requirements 549 if 'requires' in testDict: 550 for requirement in testDict['requires'].split(): 551 requirement=requirement.strip() 552 if not requirement: continue 553 if debug: print indent+"Requirement: ", requirement 554 isNull=False 555 if requirement.startswith("!"): 556 requirement=requirement[1:]; isNull=True 557 # Precision requirement for reals 558 if requirement in self.precision_types: 559 if self.conf['PETSC_PRECISION']==requirement: 560 if isNull: 561 testDict['SKIP'].append("not "+requirement+" required") 562 continue 563 continue # Success 564 elif not isNull: 565 testDict['SKIP'].append(requirement+" required") 566 continue 567 # Precision requirement for ints 568 if requirement in self.integer_types: 569 if requirement=="int32": 570 if self.conf['PETSC_SIZEOF_INT']==4: 571 if isNull: 572 testDict['SKIP'].append("not int32 required") 573 continue 574 continue # Success 575 elif not isNull: 576 testDict['SKIP'].append("int32 required") 577 continue 578 if requirement=="int64": 579 if self.conf['PETSC_SIZEOF_INT']==8: 580 if isNull: 581 testDict['SKIP'].append("NOT int64 required") 582 continue 583 continue # Success 584 elif not isNull: 585 testDict['SKIP'].append("int64 required") 586 continue 587 # Datafilespath 588 if requirement=="datafilespath" and not isNull: 589 testDict['SKIP'].append("Requires DATAFILESPATH") 590 continue 591 # Defines -- not sure I have comments matching 592 if "define(" in requirement.lower(): 593 reqdef=requirement.split("(")[1].split(")")[0] 594 if reqdef in self.conf: 595 if isNull: 596 testDict['SKIP'].append("Null requirement not met: "+requirement) 597 continue 598 continue # Success 599 elif not isNull: 600 testDict['SKIP'].append("Required: "+requirement) 601 continue 602 603 # Rest should be packages that we can just get from conf 604 if requirement == "complex": 605 petscconfvar="PETSC_USE_COMPLEX" 606 else: 607 petscconfvar="PETSC_HAVE_"+requirement.upper() 608 if self.conf.get(petscconfvar): 609 if isNull: 610 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 611 continue 612 continue # Success 613 elif not isNull: 614 if debug: print "requirement not found: ", requirement 615 testDict['SKIP'].append(petscconfvar+" requirement not met") 616 continue 617 618 return testDict['SKIP'] == [] 619 620 def genPetscTests_summarize(self,dataDict): 621 """ 622 Required method to state what happened 623 """ 624 if not self.summarize: return 625 indent=" " 626 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 627 fh=open(fhname,"w") 628 #print "See ", fhname 629 for root in dataDict: 630 relroot=self.relpath(self.petsc_dir,root) 631 pkg=relroot.split("/")[1] 632 fh.write(relroot+"\n") 633 allSrcs=[] 634 for lang in LANGS: allSrcs=allSrcs+self.sources[pkg][lang]['srcs'] 635 for exfile in dataDict[root]: 636 # Basic information 637 fullfile=os.path.join(root,exfile) 638 rfile=self.relpath(self.petsc_dir,fullfile) 639 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 640 fh.write(indent+exfile+indent*4+builtStatus+"\n") 641 642 for test in dataDict[root][exfile]: 643 if test in self.buildkeys: continue 644 line=indent*2+test 645 fh.write(line+"\n") 646 # Looks nice to have the keys in order 647 #for key in dataDict[root][exfile][test]: 648 for key in "isrun abstracted nsize args requires script".split(): 649 if key not in dataDict[root][exfile][test]: continue 650 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 651 fh.write(line+"\n") 652 fh.write("\n") 653 fh.write("\n") 654 fh.write("\n") 655 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 656 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 657 fh.close() 658 return 659 660 def genPetscTests(self,root,dirs,files,dataDict): 661 """ 662 Go through and parse the source files in the directory to generate 663 the examples based on the metadata contained in the source files 664 """ 665 debug=False 666 # Use examplesAnalyze to get what the makefles think are sources 667 #self.examplesAnalyze(root,dirs,files,anlzDict) 668 669 dataDict[root]={} 670 671 for exfile in files: 672 #TST: Until we replace files, still leaving the orginals as is 673 #if not exfile.startswith("new_"+"ex"): continue 674 if not exfile.startswith("ex"): continue 675 676 # Convenience 677 fullex=os.path.join(root,exfile) 678 relpfile=self.relpath(self.petsc_dir,fullex) 679 if debug: print relpfile 680 dataDict[root].update(testparse.parseTestFile(fullex,0)) 681 # Need to check and make sure tests are in the file 682 # if verbosity>=1: print relpfile 683 if exfile in dataDict[root]: 684 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 685 686 return 687 688 def walktree(self,top,action="printFiles"): 689 """ 690 Walk a directory tree, starting from 'top' 691 """ 692 #print "action", action 693 # Goal of action is to fill this dictionary 694 dataDict={} 695 for root, dirs, files in os.walk(top, topdown=False): 696 if not "examples" in root: continue 697 if not os.path.isfile(os.path.join(root,"makefile")): continue 698 bname=os.path.basename(root.rstrip("/")) 699 if bname=="tests" or bname=="tutorials": 700 eval("self."+action+"(root,dirs,files,dataDict)") 701 if type(top) != types.StringType: 702 raise TypeError("top must be a string") 703 # Now summarize this dictionary 704 eval("self."+action+"_summarize(dataDict)") 705 return dataDict 706 707 def gen_gnumake(self, fd): 708 """ 709 Overwrite of the method in the base PETSc class 710 """ 711 def write(stem, srcs): 712 for lang in LANGS: 713 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 714 for pkg in PKGS: 715 srcs = self.gen_pkg(pkg) 716 write('testsrcs-' + pkg, srcs) 717 return self.gendeps 718 719 def gen_pkg(self, pkg): 720 """ 721 Overwrite of the method in the base PETSc class 722 """ 723 return self.sources[pkg] 724 725 def write_gnumake(self,dataDict): 726 """ 727 Write out something similar to files from gmakegen.py 728 729 Test depends on script which also depends on source 730 file, but since I don't have a good way generating 731 acting on a single file (oops) just depend on 732 executable which in turn will depend on src file 733 """ 734 # Different options for how to set up the targets 735 compileExecsFirst=False 736 737 # Open file 738 arch_files = self.arch_path('lib','petsc','conf', 'testfiles') 739 fd = open(arch_files, 'w') 740 741 # Write out the sources 742 gendeps = self.gen_gnumake(fd) 743 744 # Write out the tests and execname targets 745 fd.write("\n#Tests and executables\n") # Delimiter 746 747 for pkg in PKGS: 748 # These grab the ones that are built 749 for lang in LANGS: 750 testdeps=[] 751 for ftest in self.tests[pkg][lang]: 752 test=os.path.basename(ftest) 753 basedir=os.path.dirname(ftest) 754 testdeps.append(self.nameSpace(test,basedir)) 755 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 756 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 757 758 # test targets 759 for ftest in self.tests[pkg][lang]: 760 test=os.path.basename(ftest) 761 basedir=os.path.dirname(ftest) 762 testdir="${TESTDIR}/"+basedir+"/" 763 nmtest=self.nameSpace(test,basedir) 764 rundir=os.path.join(testdir,test) 765 #print test, nmtest 766 script=test+".sh" 767 768 # Deps 769 exfile=self.tests[pkg][lang][ftest]['exfile'] 770 fullex=os.path.join(self.petsc_dir,exfile) 771 localexec=self.tests[pkg][lang][ftest]['exec'] 772 execname=os.path.join(testdir,localexec) 773 fullscript=os.path.join(testdir,script) 774 tmpfile=os.path.join(testdir,test,test+".tmp") 775 776 # *.counts depends on the script and either executable (will 777 # be run) or the example source file (SKIP or TODO) 778 fd.write('%s.counts : %s %s\n' 779 % (os.path.join('$(TESTDIR)/counts', nmtest), 780 fullscript, 781 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex)) 782 # Now write the args: 783 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 784 785 fd.close() 786 return 787 788 def writeHarness(self,output,dataDict): 789 """ 790 This is set up to write out multiple harness even if only gnumake 791 is supported now 792 """ 793 eval("self.write_"+output+"(dataDict)") 794 return 795 796def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False): 797 if output is None: 798 output = 'gnumake' 799 800 801 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, verbose=verbose, single_ex=single_ex) 802 dataDict=pEx.walktree(os.path.join(pEx.petsc_dir,'src'),action="genPetscTests") 803 pEx.writeHarness(output,dataDict) 804 805if __name__ == '__main__': 806 import optparse 807 parser = optparse.OptionParser() 808 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 809 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 810 parser.add_option('--output', help='Location to write output file', default=None) 811 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 812 opts, extra_args = parser.parse_args() 813 if extra_args: 814 import sys 815 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 816 exit(1) 817 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, single_ex=opts.single_executable) 818