1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 115 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 116 nameString=prefix+srcfile 117 else: 118 #nameString=srcdir+": "+srcfile 119 nameString=srcfile 120 return nameString 121 122 def getLanguage(self,srcfile): 123 """ 124 Based on the source, determine associated language as found in gmakegen.LANGS 125 Can we just return srcext[1:\] now? 126 """ 127 langReq=None 128 srcext=os.path.splitext(srcfile)[-1] 129 if srcext in ".F90".split(): langReq="F90" 130 if srcext in ".F".split(): langReq="F" 131 if srcext in ".cxx".split(): langReq="cxx" 132 if srcext == ".cu": langReq="cu" 133 if srcext == ".c": langReq="c" 134 #if not langReq: print "ERROR: ", srcext, srcfile 135 return langReq 136 137 def _getLoopVars(self,inDict,testname, isSubtest=False): 138 """ 139 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 140 Return: 141 inDict['args']: -ksp_monitor 142 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 143 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 144 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 145 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 146 subst should be passed in instead of inDict 147 """ 148 loopVars={}; newargs="" 149 lkeys=inDict.keys() 150 lsuffix='_' 151 from testparse import parseLoopArgs 152 for key in lkeys: 153 if type(inDict[key])!=types.StringType: continue 154 keystr = str(inDict[key]) 155 akey=('subargs' if key=='args' else key) # what to assign 156 if akey not in inDict: inDict[akey]='' 157 varlist=[] 158 for varset in re.split('-(?=[a-zA-Z])',keystr): 159 if not varset.strip(): continue 160 if '{{' in varset: 161 keyvar,lvars,ftype=parseLoopArgs(varset) 162 if akey not in loopVars: loopVars[akey]={} 163 varlist.append(keyvar) 164 loopVars[akey][keyvar]=[keyvar,lvars] 165 if akey=='nsize': 166 inDict[akey] = '${' + keyvar + '}' 167 lsuffix+=akey+'-'+inDict[akey]+'_' 168 else: 169 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 170 lsuffix+=keyvar+'-${' + keyvar + '}_' 171 else: 172 if key=='args': newargs+=" -"+varset.strip() 173 if varlist: loopVars[akey]['varlist']=varlist 174 175 176 # For subtests, args are always substituted in (not top level) 177 if isSubtest: 178 inDict['subargs']+=" "+newargs.strip() 179 inDict['args']='' 180 if 'label_suffix' in inDict: 181 inDict['label_suffix']+=lsuffix.rstrip('_') 182 else: 183 inDict['label_suffix']=lsuffix.rstrip('_') 184 else: 185 if loopVars.keys(): 186 inDict['args']=newargs.strip() 187 inDict['label_suffix']=lsuffix.rstrip('_') 188 if loopVars.keys(): 189 return loopVars 190 else: 191 return None 192 193 def getArgLabel(self,testDict): 194 """ 195 In all of the arguments in the test dictionary, create a simple 196 string for searching within the makefile system. For simplicity in 197 search, remove "-", for strings, etc. 198 Also, concatenate the arg commands 199 For now, ignore nsize -- seems hard to search for anyway 200 """ 201 # Collect all of the args associated with a test 202 argStr=("" if 'args' not in testDict else testDict['args']) 203 if 'subtests' in testDict: 204 for stest in testDict["subtests"]: 205 sd=testDict[stest] 206 argStr=argStr+("" if 'args' not in sd else sd['args']) 207 208 # Now go through and cleanup 209 argStr=re.sub('{{(.*?)}}',"",argStr) 210 argStr=re.sub('-'," ",argStr) 211 for digit in string.digits: argStr=re.sub(digit," ",argStr) 212 argStr=re.sub("\.","",argStr) 213 argStr=re.sub(",","",argStr) 214 argStr=re.sub('\+',' ',argStr) 215 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 216 return argStr.strip() 217 218 def addToSources(self,exfile,root,srcDict): 219 """ 220 Put into data structure that allows easy generation of makefile 221 """ 222 rpath=self.srcrelpath(root) 223 pkg=rpath.split(os.path.sep)[1] 224 relpfile=os.path.join(rpath,exfile) 225 lang=self.getLanguage(exfile) 226 if not lang: return 227 self.sources[pkg][lang]['srcs'].append(relpfile) 228 if 'depends' in srcDict: 229 depSrc=srcDict['depends'] 230 depObj=os.path.splitext(depSrc)[0]+".o" 231 self.sources[pkg][lang][relpfile]=os.path.join(rpath,depObj) 232 233 # In gmakefile, ${TESTDIR} var specifies the object compilation 234 testsdir=self.srcrelpath(root)+"/" 235 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 236 self.objects[pkg].append(objfile) 237 return 238 239 def addToTests(self,test,root,exfile,execname,testDict): 240 """ 241 Put into data structure that allows easy generation of makefile 242 Organized by languages to allow testing of languages 243 """ 244 rpath=self.srcrelpath(root) 245 pkg=rpath.split("/")[1] 246 #nmtest=self.nameSpace(test,root) 247 nmtest=os.path.join(rpath,test) 248 lang=self.getLanguage(exfile) 249 if not lang: return 250 self.tests[pkg][lang][nmtest]={} 251 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 252 self.tests[pkg][lang][nmtest]['exec']=execname 253 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 254 return 255 256 def getExecname(self,exfile,root): 257 """ 258 Generate bash script using template found next to this file. 259 This file is read in at constructor time to avoid file I/O 260 """ 261 rpath=self.srcrelpath(root) 262 if self.single_ex: 263 execname=rpath.split("/")[1]+"-ex" 264 else: 265 execname=os.path.splitext(exfile)[0] 266 return execname 267 268 def getSubstVars(self,testDict,rpath,testname): 269 """ 270 Create a dictionary with all of the variables that get substituted 271 into the template commands found in example_template.py 272 """ 273 subst={} 274 275 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 276 if 'nsize' not in testDict: testDict['nsize']=1 277 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 278 for ak in testparse.acceptedkeys: 279 if ak=='test': continue 280 subst[ak]=(testDict[ak] if ak in testDict else '') 281 282 # Now do other variables 283 subst['execname']=testDict['execname'] 284 if 'filter' in testDict: 285 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 286 287 # Others 288 subst['subargs']='' # Default. For variables override 289 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 290 subst['label_suffix']='' 291 subst['comments']="\n#".join(subst['comments'].split("\n")) 292 if subst['comments']: subst['comments']="#"+subst['comments'] 293 subst['exec']="../"+subst['execname'] 294 subst['testroot']=self.testroot_dir 295 subst['testname']=testname 296 dp = self.conf.get('DATAFILESPATH','') 297 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 298 299 # This is used to label some matrices 300 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 301 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 302 303 # These can have for loops and are treated separately later 304 subst['nsize']=str(subst['nsize']) 305 306 #Conf vars 307 if self.petsc_arch.find('valgrind')>=0: 308 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 309 else: 310 subst['mpiexec']=self.conf['MPIEXEC'] 311 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 312 subst['petsc_arch']=self.petsc_arch 313 if not self.inInstallDir: 314 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 315 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 316 else: 317 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 318 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 319 subst['diff']=self.conf['DIFF'] 320 subst['rm']=self.conf['RM'] 321 subst['grep']=self.conf['GREP'] 322 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 323 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 324 325 # Output file is special because of subtests override 326 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 327 if not "_" in defroot: defroot=defroot+"_1" 328 subst['defroot']=defroot 329 subst['label']=self.nameSpace(defroot,subst['srcdir']) 330 subst['redirect_file']=defroot+".tmp" 331 if 'output_file' not in testDict: 332 subst['output_file']="output/"+defroot+".out" 333 # Add in the full path here. 334 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 335 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 336 if not subst['TODO']: 337 print "Warning: "+subst['output_file']+" not found." 338 # Worry about alt files here -- see 339 # src/snes/examples/tutorials/output/ex22*.out 340 altlist=[subst['output_file']] 341 for i in range(1,3): 342 altroot=defroot+"_alt" 343 if i==2: altroot=altroot+"_2" 344 af="output/"+altroot+".out" 345 srcaf=os.path.join(subst['srcdir'],af) 346 fullaf=os.path.join(self.petsc_dir,srcaf) 347 if os.path.isfile(fullaf): altlist.append(srcaf) 348 if len(altlist)>1: subst['altfiles']=altlist 349 #if len(altlist)>1: print "Found alt files: ",altlist 350 351 return subst 352 353 def getCmds(self,subst,i): 354 """ 355 Generate bash script using template found next to this file. 356 This file is read in at constructor time to avoid file I/O 357 """ 358 nindnt=i # the start and has to be consistent with below 359 cmdindnt=self.indent*nindnt 360 cmdLines="" 361 362 # MPI is the default -- but we have a few odd commands 363 if not subst['command']: 364 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 365 else: 366 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 367 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 368 369 cmdLines+=cmdindnt+'if test $res = 0; then\n' 370 diffindnt=self.indent*(nindnt+1) 371 if not subst['filter_output']: 372 if 'altfiles' not in subst: 373 cmd=diffindnt+self._substVars(subst,example_template.difftest) 374 else: 375 # Have to do it by hand a bit because of variable number of alt files 376 rf=subst['redirect_file'] 377 cmd=diffindnt+example_template.difftest.split('@')[0] 378 for i in range(len(subst['altfiles'])): 379 af=subst['altfiles'][i] 380 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 381 if i!=len(subst['altfiles'])-1: 382 cmd+=' || ${diff_exe} ' 383 else: 384 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 385 cmd+=subst['label_suffix']+' ""' # Quotes are painful 386 else: 387 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 388 cmdLines+=cmd+"\n" 389 cmdLines+=cmdindnt+'else\n' 390 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 391 cmdLines+=cmdindnt+'fi\n' 392 return cmdLines 393 394 def _substVars(self,subst,origStr): 395 """ 396 Substitute variables 397 """ 398 Str=origStr 399 for subkey in subst: 400 if type(subst[subkey])!=types.StringType: continue 401 patt="@"+subkey.upper()+"@" 402 Str=re.sub(patt,subst[subkey],Str) 403 return Str 404 405 def _writeTodoSkip(self,fh,tors,reasons,footer): 406 """ 407 Write out the TODO and SKIP lines in the file 408 The TODO or SKIP variable, tors, should be lower case 409 """ 410 TORS=tors.upper() 411 template=eval("example_template."+tors+"line") 412 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 413 tab = '' 414 if reasons: 415 fh.write('if ! $force; then\n') 416 tab = tab + ' ' 417 if reasons == ["Requires DATAFILESPATH"]: 418 # The only reason not to run is DATAFILESPATH, which we check at run-time 419 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 420 tab = tab + ' ' 421 if reasons: 422 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 423 fh.write(tab+footer+"\n") 424 fh.write(tab+"exit\n") 425 if reasons == ["Requires DATAFILESPATH"]: 426 fh.write(' fi\n') 427 if reasons: 428 fh.write('fi\n') 429 fh.write('\n\n') 430 return 431 432 def getLoopVarsHead(self,loopVars,i): 433 """ 434 Generate a nicely indented string with the format loops 435 Here is what the data structure looks like 436 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 437 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 438 loopVars['subargs']['pc_type']=["j","cholesky sor"] 439 """ 440 outstr=''; indnt=self.indent 441 for key in loopVars: 442 for var in loopVars[key]['varlist']: 443 varval=loopVars[key][var] 444 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 445 i = i + 1 446 return (outstr,i) 447 448 def getLoopVarsFoot(self,loopVars,i): 449 outstr=''; indnt=self.indent 450 for key in loopVars: 451 for var in loopVars[key]['varlist']: 452 i = i - 1 453 outstr += indnt * i + "done\n" 454 return (outstr,i) 455 456 def genRunScript(self,testname,root,isRun,srcDict): 457 """ 458 Generate bash script using template found next to this file. 459 This file is read in at constructor time to avoid file I/O 460 """ 461 # runscript_dir directory has to be consistent with gmakefile 462 testDict=srcDict[testname] 463 rpath=self.srcrelpath(root) 464 runscript_dir=os.path.join(self.testroot_dir,rpath) 465 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 466 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 467 petscvarfile=os.path.join(self.arch_dir,'lib','petsc','conf','petscvariables') 468 469 # Get variables to go into shell scripts. last time testDict used 470 subst=self.getSubstVars(testDict,rpath,testname) 471 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 472 #if '33_' in testname: print subst['subargs'] 473 474 #Handle runfiles 475 for lfile in subst.get('localrunfiles','').split(): 476 fullfile=os.path.join(root,lfile) 477 shutil.copy(fullfile,runscript_dir) 478 # Check subtests for local runfiles 479 for stest in subst.get("subtests",[]): 480 for lfile in testDict[stest].get('localrunfiles','').split(): 481 fullfile=os.path.join(root,lfile) 482 shutil.copy(fullfile,self.runscript_dir) 483 484 # Now substitute the key variables into the header and footer 485 header=self._substVars(subst,example_template.header) 486 # The header is done twice to enable @...@ in header 487 header=self._substVars(subst,header) 488 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 489 490 # Start writing the file 491 fh.write(header+"\n") 492 493 # If there is a TODO or a SKIP then we do it before writing out the 494 # rest of the command (which is useful for working on the test) 495 # SKIP and TODO can be for the source file or for the runs 496 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 497 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 498 499 j=0 # for indentation 500 501 if loopVars: 502 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 503 if (loopHead): fh.write(loopHead+"\n") 504 505 # Subtests are special 506 if 'subtests' in testDict: 507 substP=subst # Subtests can inherit args but be careful 508 k=0 # for label suffixes 509 for stest in testDict["subtests"]: 510 subst=substP.copy() 511 subst.update(testDict[stest]) 512 # nsize is special because it is usually overwritten 513 if 'nsize' in testDict[stest]: 514 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 515 else: 516 fh.write("nsize=1\n") 517 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 518 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 519 #if '10_9' in testname: print sLoopVars 520 if sLoopVars: 521 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 522 fh.write(sLoopHead+"\n") 523 fh.write(self.getCmds(subst,j)+"\n") 524 if sLoopVars: 525 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 526 fh.write(sLoopFoot+"\n") 527 else: 528 fh.write(self.getCmds(subst,j)+"\n") 529 530 if loopVars: 531 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 532 fh.write(loopFoot+"\n") 533 534 fh.write(footer+"\n") 535 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 536 #if '10_9' in testname: sys.exit() 537 return 538 539 def genScriptsAndInfo(self,exfile,root,srcDict): 540 """ 541 Generate scripts from the source file, determine if built, etc. 542 For every test in the exfile with info in the srcDict: 543 1. Determine if it needs to be run for this arch 544 2. Generate the script 545 3. Generate the data needed to write out the makefile in a 546 convenient way 547 All tests are *always* run, but some may be SKIP'd per the TAP standard 548 """ 549 debug=False 550 execname=self.getExecname(exfile,root) 551 isBuilt=self._isBuilt(exfile,srcDict) 552 for test in srcDict: 553 if test in self.buildkeys: continue 554 if debug: print self.nameSpace(exfile,root), test 555 srcDict[test]['execname']=execname # Convenience in generating scripts 556 isRun=self._isRun(srcDict[test]) 557 self.genRunScript(test,root,isRun,srcDict) 558 srcDict[test]['isrun']=isRun 559 self.addToTests(test,root,exfile,execname,srcDict[test]) 560 561 # This adds to datastructure for building deps 562 if isBuilt: self.addToSources(exfile,root,srcDict) 563 return 564 565 def _isBuilt(self,exfile,srcDict): 566 """ 567 Determine if this file should be built. 568 """ 569 # Get the language based on file extension 570 srcDict['SKIP'] = [] 571 lang=self.getLanguage(exfile) 572 if (lang=="F" or lang=="F90"): 573 if not self.have_fortran: 574 srcDict["SKIP"].append("Fortran required for this test") 575 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 576 srcDict["SKIP"].append("Fortran f90freeform required for this test") 577 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 578 srcDict["SKIP"].append("CUDA required for this test") 579 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 580 srcDict["SKIP"].append("C++ required for this test") 581 582 # Deprecated source files 583 if srcDict.get("TODO"): 584 return False 585 586 # isRun can work with srcDict to handle the requires 587 if "requires" in srcDict: 588 if srcDict["requires"]: 589 return self._isRun(srcDict) 590 591 return srcDict['SKIP'] == [] 592 593 594 def _isRun(self,testDict): 595 """ 596 Based on the requirements listed in the src file and the petscconf.h 597 info, determine whether this test should be run or not. 598 """ 599 indent=" " 600 debug=False 601 602 if 'SKIP' not in testDict: 603 testDict['SKIP'] = [] 604 # MPI requirements 605 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 606 if debug: print indent+"Cannot run parallel tests" 607 testDict['SKIP'].append("Parallel test with serial build") 608 609 # The requirements for the test are the sum of all the run subtests 610 if 'subtests' in testDict: 611 if 'requires' not in testDict: testDict['requires']="" 612 for stest in testDict['subtests']: 613 if 'requires' in testDict[stest]: 614 testDict['requires']+=" "+testDict[stest]['requires'] 615 616 617 # Now go through all requirements 618 if 'requires' in testDict: 619 for requirement in testDict['requires'].split(): 620 requirement=requirement.strip() 621 if not requirement: continue 622 if debug: print indent+"Requirement: ", requirement 623 isNull=False 624 if requirement.startswith("!"): 625 requirement=requirement[1:]; isNull=True 626 # Precision requirement for reals 627 if requirement in self.precision_types: 628 if self.conf['PETSC_PRECISION']==requirement: 629 if isNull: 630 testDict['SKIP'].append("not "+requirement+" required") 631 continue 632 continue # Success 633 elif not isNull: 634 testDict['SKIP'].append(requirement+" required") 635 continue 636 # Precision requirement for ints 637 if requirement in self.integer_types: 638 if requirement=="int32": 639 if self.conf['PETSC_SIZEOF_INT']==4: 640 if isNull: 641 testDict['SKIP'].append("not int32 required") 642 continue 643 continue # Success 644 elif not isNull: 645 testDict['SKIP'].append("int32 required") 646 continue 647 if requirement=="int64": 648 if self.conf['PETSC_SIZEOF_INT']==8: 649 if isNull: 650 testDict['SKIP'].append("NOT int64 required") 651 continue 652 continue # Success 653 elif not isNull: 654 testDict['SKIP'].append("int64 required") 655 continue 656 # Datafilespath 657 if requirement=="datafilespath" and not isNull: 658 testDict['SKIP'].append("Requires DATAFILESPATH") 659 continue 660 # Defines -- not sure I have comments matching 661 if "define(" in requirement.lower(): 662 reqdef=requirement.split("(")[1].split(")")[0] 663 if reqdef in self.conf: 664 if isNull: 665 testDict['SKIP'].append("Null requirement not met: "+requirement) 666 continue 667 continue # Success 668 elif not isNull: 669 testDict['SKIP'].append("Required: "+requirement) 670 continue 671 672 # Rest should be packages that we can just get from conf 673 if requirement == "complex": 674 petscconfvar="PETSC_USE_COMPLEX" 675 else: 676 petscconfvar="PETSC_HAVE_"+requirement.upper() 677 if self.conf.get(petscconfvar): 678 if isNull: 679 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 680 continue 681 continue # Success 682 elif not isNull: 683 if debug: print "requirement not found: ", requirement 684 testDict['SKIP'].append(petscconfvar+" requirement not met") 685 continue 686 687 return testDict['SKIP'] == [] 688 689 def genPetscTests_summarize(self,dataDict): 690 """ 691 Required method to state what happened 692 """ 693 if not self.summarize: return 694 indent=" " 695 print(self.testroot_dir, os.path.realpath(os.curdir)) 696 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 697 fh=open(fhname,"w") 698 #print "See ", fhname 699 for root in dataDict: 700 relroot=self.srcrelpath(root) 701 pkg=relroot.split("/")[1] 702 fh.write(relroot+"\n") 703 allSrcs=[] 704 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 705 for exfile in dataDict[root]: 706 # Basic information 707 rfile=os.path.join(relroot,exfile) 708 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 709 fh.write(indent+exfile+indent*4+builtStatus+"\n") 710 711 for test in dataDict[root][exfile]: 712 if test in self.buildkeys: continue 713 line=indent*2+test 714 fh.write(line+"\n") 715 # Looks nice to have the keys in order 716 #for key in dataDict[root][exfile][test]: 717 for key in "isrun abstracted nsize args requires script".split(): 718 if key not in dataDict[root][exfile][test]: continue 719 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 720 fh.write(line+"\n") 721 fh.write("\n") 722 fh.write("\n") 723 fh.write("\n") 724 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 725 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 726 fh.close() 727 return 728 729 def genPetscTests(self,root,dirs,files,dataDict): 730 """ 731 Go through and parse the source files in the directory to generate 732 the examples based on the metadata contained in the source files 733 """ 734 debug=False 735 # Use examplesAnalyze to get what the makefles think are sources 736 #self.examplesAnalyze(root,dirs,files,anlzDict) 737 738 dataDict[root]={} 739 740 for exfile in files: 741 #TST: Until we replace files, still leaving the orginals as is 742 #if not exfile.startswith("new_"+"ex"): continue 743 #if not exfile.startswith("ex"): continue 744 745 # Ignore emacs files 746 if exfile.startswith("#") or exfile.startswith(".#"): continue 747 748 # Convenience 749 fullex=os.path.join(root,exfile) 750 if self.verbose: print(' --> '+fullex) 751 dataDict[root].update(testparse.parseTestFile(fullex,0)) 752 if exfile in dataDict[root]: 753 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 754 755 return 756 757 def walktree(self,top): 758 """ 759 Walk a directory tree, starting from 'top' 760 """ 761 #print "action", action 762 # Goal of action is to fill this dictionary 763 dataDict={} 764 for root, dirs, files in os.walk(top, topdown=False): 765 if not "examples" in root: continue 766 if not os.path.isfile(os.path.join(root,"makefile")): continue 767 if self.verbose: print(root) 768 bname=os.path.basename(root.rstrip("/")) 769 self.genPetscTests(root,dirs,files,dataDict) 770 # Now summarize this dictionary 771 self.genPetscTests_summarize(dataDict) 772 return dataDict 773 774 def gen_gnumake(self, fd): 775 """ 776 Overwrite of the method in the base PETSc class 777 """ 778 def write(stem, srcs): 779 for lang in LANGS: 780 if srcs[lang]['srcs']: 781 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 782 for pkg in PKGS: 783 srcs = self.gen_pkg(pkg) 784 write('testsrcs-' + pkg, srcs) 785 # Handle dependencies 786 for lang in LANGS: 787 for exfile in srcs[lang]['srcs']: 788 if exfile in srcs[lang]: 789 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 790 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 791 fd.write(exfile+": $(TESTDIR)/"+ srcs[lang][exfile]+'\n') 792 fd.write(ex +": "+exfo+" $(TESTDIR)/"+srcs[lang][exfile] +'\n') 793 794 return self.gendeps 795 796 def gen_pkg(self, pkg): 797 """ 798 Overwrite of the method in the base PETSc class 799 """ 800 return self.sources[pkg] 801 802 def write_gnumake(self,dataDict): 803 """ 804 Write out something similar to files from gmakegen.py 805 806 Test depends on script which also depends on source 807 file, but since I don't have a good way generating 808 acting on a single file (oops) just depend on 809 executable which in turn will depend on src file 810 """ 811 # Different options for how to set up the targets 812 compileExecsFirst=False 813 814 # Open file 815 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 816 fd = open(arch_files, 'w') 817 818 # Write out the sources 819 gendeps = self.gen_gnumake(fd) 820 821 # Write out the tests and execname targets 822 fd.write("\n#Tests and executables\n") # Delimiter 823 824 for pkg in PKGS: 825 # These grab the ones that are built 826 for lang in LANGS: 827 testdeps=[] 828 for ftest in self.tests[pkg][lang]: 829 test=os.path.basename(ftest) 830 basedir=os.path.dirname(ftest) 831 testdeps.append(self.nameSpace(test,basedir)) 832 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 833 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 834 835 # test targets 836 for ftest in self.tests[pkg][lang]: 837 test=os.path.basename(ftest) 838 basedir=os.path.dirname(ftest) 839 testdir="${TESTDIR}/"+basedir+"/" 840 nmtest=self.nameSpace(test,basedir) 841 rundir=os.path.join(testdir,test) 842 #print test, nmtest 843 script=test+".sh" 844 845 # Deps 846 exfile=self.tests[pkg][lang][ftest]['exfile'] 847 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 848 localexec=self.tests[pkg][lang][ftest]['exec'] 849 execname=os.path.join(testdir,localexec) 850 fullscript=os.path.join(testdir,script) 851 tmpfile=os.path.join(testdir,test,test+".tmp") 852 853 # *.counts depends on the script and either executable (will 854 # be run) or the example source file (SKIP or TODO) 855 if exfile in self.sources[pkg][lang]: 856 fd.write('%s.counts : %s %s %s\n' 857 % (os.path.join('$(TESTDIR)/counts', nmtest), 858 fullscript, 859 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex, 860 os.path.join('$(TESTDIR)',self.sources[pkg][lang][exfile])) 861 ) 862 else: 863 fd.write('%s.counts : %s %s\n' 864 % (os.path.join('$(TESTDIR)/counts', nmtest), 865 fullscript, 866 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 867 ) 868 869 # Now write the args: 870 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 871 872 fd.close() 873 return 874 875 def writeHarness(self,output,dataDict): 876 """ 877 This is set up to write out multiple harness even if only gnumake 878 is supported now 879 """ 880 eval("self.write_"+output+"(dataDict)") 881 return 882 883def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 884 if output is None: 885 output = 'gnumake' 886 887 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 888 if petsc_arch: 889 if len(petsc_arch.split(os.path.sep))>1: 890 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 891 892 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 893 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 894 testdir=testdir) 895 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 896 pEx.writeHarness(output,dataDict) 897 898if __name__ == '__main__': 899 import optparse 900 parser = optparse.OptionParser() 901 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 902 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 903 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 904 parser.add_option('--output', help='Location to write output file', default=None) 905 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 906 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 907 opts, extra_args = parser.parse_args() 908 opts, extra_args = parser.parse_args() 909 if extra_args: 910 import sys 911 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 912 exit(1) 913 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 914 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 915