1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 115 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 116 nameString=prefix+srcfile 117 else: 118 #nameString=srcdir+": "+srcfile 119 nameString=srcfile 120 return nameString 121 122 def getLanguage(self,srcfile): 123 """ 124 Based on the source, determine associated language as found in gmakegen.LANGS 125 Can we just return srcext[1:\] now? 126 """ 127 langReq=None 128 srcext=os.path.splitext(srcfile)[-1] 129 if srcext in ".F90".split(): langReq="F90" 130 if srcext in ".F".split(): langReq="F" 131 if srcext in ".cxx".split(): langReq="cxx" 132 if srcext == ".cu": langReq="cu" 133 if srcext == ".c": langReq="c" 134 #if not langReq: print "ERROR: ", srcext, srcfile 135 return langReq 136 137 def _getLoopVars(self,inDict,testname, isSubtest=False): 138 """ 139 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 140 Return: 141 inDict['args']: -ksp_monitor 142 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 143 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 144 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 145 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 146 subst should be passed in instead of inDict 147 """ 148 loopVars={}; newargs="" 149 lkeys=inDict.keys() 150 lsuffix='_' 151 from testparse import parseLoopArgs 152 for key in lkeys: 153 if type(inDict[key])!=types.StringType: continue 154 keystr = str(inDict[key]) 155 akey=('subargs' if key=='args' else key) # what to assign 156 if akey not in inDict: inDict[akey]='' 157 varlist=[] 158 for varset in re.split('-(?=[a-zA-Z])',keystr): 159 if not varset.strip(): continue 160 if '{{' in varset: 161 keyvar,lvars,ftype=parseLoopArgs(varset) 162 if akey not in loopVars: loopVars[akey]={} 163 varlist.append(keyvar) 164 loopVars[akey][keyvar]=[keyvar,lvars] 165 if akey=='nsize': 166 inDict[akey] = '${' + keyvar + '}' 167 lsuffix+=akey+'-'+inDict[akey]+'_' 168 else: 169 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 170 lsuffix+=keyvar+'-${' + keyvar + '}_' 171 else: 172 if key=='args': newargs+=" -"+varset.strip() 173 if varlist: loopVars[akey]['varlist']=varlist 174 175 176 # For subtests, args are always substituted in (not top level) 177 if isSubtest: 178 inDict['subargs']+=" "+newargs.strip() 179 inDict['args']='' 180 if 'label_suffix' in inDict: 181 inDict['label_suffix']+=lsuffix.rstrip('_') 182 else: 183 inDict['label_suffix']=lsuffix.rstrip('_') 184 else: 185 if loopVars.keys(): 186 inDict['args']=newargs.strip() 187 inDict['label_suffix']=lsuffix.rstrip('_') 188 if loopVars.keys(): 189 return loopVars 190 else: 191 return None 192 193 def getArgLabel(self,testDict): 194 """ 195 In all of the arguments in the test dictionary, create a simple 196 string for searching within the makefile system. For simplicity in 197 search, remove "-", for strings, etc. 198 Also, concatenate the arg commands 199 For now, ignore nsize -- seems hard to search for anyway 200 """ 201 # Collect all of the args associated with a test 202 argStr=("" if 'args' not in testDict else testDict['args']) 203 if 'subtests' in testDict: 204 for stest in testDict["subtests"]: 205 sd=testDict[stest] 206 argStr=argStr+("" if 'args' not in sd else sd['args']) 207 208 # Now go through and cleanup 209 argStr=re.sub('{{(.*?)}}',"",argStr) 210 argStr=re.sub('-'," ",argStr) 211 for digit in string.digits: argStr=re.sub(digit," ",argStr) 212 argStr=re.sub("\.","",argStr) 213 argStr=re.sub(",","",argStr) 214 argStr=re.sub('\+',' ',argStr) 215 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 216 return argStr.strip() 217 218 def addToSources(self,exfile,root,srcDict): 219 """ 220 Put into data structure that allows easy generation of makefile 221 """ 222 rpath=self.srcrelpath(root) 223 pkg=rpath.split(os.path.sep)[1] 224 relpfile=os.path.join(rpath,exfile) 225 lang=self.getLanguage(exfile) 226 if not lang: return 227 self.sources[pkg][lang]['srcs'].append(relpfile) 228 self.sources[pkg][lang][relpfile] = [] 229 if 'depends' in srcDict: 230 depSrcList=srcDict['depends'].split() 231 for depSrc in depSrcList: 232 depObj=os.path.splitext(depSrc)[0]+".o" 233 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 234 235 # In gmakefile, ${TESTDIR} var specifies the object compilation 236 testsdir=self.srcrelpath(root)+"/" 237 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 238 self.objects[pkg].append(objfile) 239 return 240 241 def addToTests(self,test,root,exfile,execname,testDict): 242 """ 243 Put into data structure that allows easy generation of makefile 244 Organized by languages to allow testing of languages 245 """ 246 rpath=self.srcrelpath(root) 247 pkg=rpath.split("/")[1] 248 #nmtest=self.nameSpace(test,root) 249 nmtest=os.path.join(rpath,test) 250 lang=self.getLanguage(exfile) 251 if not lang: return 252 self.tests[pkg][lang][nmtest]={} 253 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 254 self.tests[pkg][lang][nmtest]['exec']=execname 255 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 256 return 257 258 def getExecname(self,exfile,root): 259 """ 260 Generate bash script using template found next to this file. 261 This file is read in at constructor time to avoid file I/O 262 """ 263 rpath=self.srcrelpath(root) 264 if self.single_ex: 265 execname=rpath.split("/")[1]+"-ex" 266 else: 267 execname=os.path.splitext(exfile)[0] 268 return execname 269 270 def getSubstVars(self,testDict,rpath,testname): 271 """ 272 Create a dictionary with all of the variables that get substituted 273 into the template commands found in example_template.py 274 """ 275 subst={} 276 277 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 278 if 'nsize' not in testDict: testDict['nsize']=1 279 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 280 for ak in testparse.acceptedkeys: 281 if ak=='test': continue 282 subst[ak]=(testDict[ak] if ak in testDict else '') 283 284 # Now do other variables 285 subst['execname']=testDict['execname'] 286 if 'filter' in testDict: 287 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 288 289 # Others 290 subst['subargs']='' # Default. For variables override 291 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 292 subst['label_suffix']='' 293 subst['comments']="\n#".join(subst['comments'].split("\n")) 294 if subst['comments']: subst['comments']="#"+subst['comments'] 295 subst['exec']="../"+subst['execname'] 296 subst['testroot']=self.testroot_dir 297 subst['testname']=testname 298 dp = self.conf.get('DATAFILESPATH','') 299 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 300 301 # This is used to label some matrices 302 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 303 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 304 305 # These can have for loops and are treated separately later 306 subst['nsize']=str(subst['nsize']) 307 308 #Conf vars 309 if self.petsc_arch.find('valgrind')>=0: 310 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 311 else: 312 subst['mpiexec']=self.conf['MPIEXEC'] 313 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 314 subst['petsc_arch']=self.petsc_arch 315 if not self.inInstallDir: 316 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 317 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 318 else: 319 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 320 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 321 subst['diff']=self.conf['DIFF'] 322 subst['rm']=self.conf['RM'] 323 subst['grep']=self.conf['GREP'] 324 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 325 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 326 327 # Output file is special because of subtests override 328 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 329 if not "_" in defroot: defroot=defroot+"_1" 330 subst['defroot']=defroot 331 subst['label']=self.nameSpace(defroot,subst['srcdir']) 332 subst['redirect_file']=defroot+".tmp" 333 if 'output_file' not in testDict: 334 subst['output_file']="output/"+defroot+".out" 335 # Add in the full path here. 336 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 337 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 338 if not subst['TODO']: 339 print "Warning: "+subst['output_file']+" not found." 340 # Worry about alt files here -- see 341 # src/snes/examples/tutorials/output/ex22*.out 342 altlist=[subst['output_file']] 343 for i in range(1,3): 344 altroot=defroot+"_alt" 345 if i==2: altroot=altroot+"_2" 346 af="output/"+altroot+".out" 347 srcaf=os.path.join(subst['srcdir'],af) 348 fullaf=os.path.join(self.petsc_dir,srcaf) 349 if os.path.isfile(fullaf): altlist.append(srcaf) 350 if len(altlist)>1: subst['altfiles']=altlist 351 #if len(altlist)>1: print "Found alt files: ",altlist 352 353 return subst 354 355 def getCmds(self,subst,i): 356 """ 357 Generate bash script using template found next to this file. 358 This file is read in at constructor time to avoid file I/O 359 """ 360 nindnt=i # the start and has to be consistent with below 361 cmdindnt=self.indent*nindnt 362 cmdLines="" 363 364 # MPI is the default -- but we have a few odd commands 365 if not subst['command']: 366 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 367 else: 368 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 369 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 370 371 cmdLines+=cmdindnt+'if test $res = 0; then\n' 372 diffindnt=self.indent*(nindnt+1) 373 if not subst['filter_output']: 374 if 'altfiles' not in subst: 375 cmd=diffindnt+self._substVars(subst,example_template.difftest) 376 else: 377 # Have to do it by hand a bit because of variable number of alt files 378 rf=subst['redirect_file'] 379 cmd=diffindnt+example_template.difftest.split('@')[0] 380 for i in range(len(subst['altfiles'])): 381 af=subst['altfiles'][i] 382 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 383 if i!=len(subst['altfiles'])-1: 384 cmd+=' || ${diff_exe} ' 385 else: 386 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 387 cmd+=subst['label_suffix']+' ""' # Quotes are painful 388 else: 389 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 390 cmdLines+=cmd+"\n" 391 cmdLines+=cmdindnt+'else\n' 392 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 393 cmdLines+=cmdindnt+'fi\n' 394 return cmdLines 395 396 def _substVars(self,subst,origStr): 397 """ 398 Substitute variables 399 """ 400 Str=origStr 401 for subkey in subst: 402 if type(subst[subkey])!=types.StringType: continue 403 patt="@"+subkey.upper()+"@" 404 Str=re.sub(patt,subst[subkey],Str) 405 return Str 406 407 def _writeTodoSkip(self,fh,tors,reasons,footer): 408 """ 409 Write out the TODO and SKIP lines in the file 410 The TODO or SKIP variable, tors, should be lower case 411 """ 412 TORS=tors.upper() 413 template=eval("example_template."+tors+"line") 414 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 415 tab = '' 416 if reasons: 417 fh.write('if ! $force; then\n') 418 tab = tab + ' ' 419 if reasons == ["Requires DATAFILESPATH"]: 420 # The only reason not to run is DATAFILESPATH, which we check at run-time 421 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 422 tab = tab + ' ' 423 if reasons: 424 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 425 fh.write(tab+footer+"\n") 426 fh.write(tab+"exit\n") 427 if reasons == ["Requires DATAFILESPATH"]: 428 fh.write(' fi\n') 429 if reasons: 430 fh.write('fi\n') 431 fh.write('\n\n') 432 return 433 434 def getLoopVarsHead(self,loopVars,i): 435 """ 436 Generate a nicely indented string with the format loops 437 Here is what the data structure looks like 438 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 439 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 440 loopVars['subargs']['pc_type']=["j","cholesky sor"] 441 """ 442 outstr=''; indnt=self.indent 443 for key in loopVars: 444 for var in loopVars[key]['varlist']: 445 varval=loopVars[key][var] 446 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 447 i = i + 1 448 return (outstr,i) 449 450 def getLoopVarsFoot(self,loopVars,i): 451 outstr=''; indnt=self.indent 452 for key in loopVars: 453 for var in loopVars[key]['varlist']: 454 i = i - 1 455 outstr += indnt * i + "done\n" 456 return (outstr,i) 457 458 def genRunScript(self,testname,root,isRun,srcDict): 459 """ 460 Generate bash script using template found next to this file. 461 This file is read in at constructor time to avoid file I/O 462 """ 463 # runscript_dir directory has to be consistent with gmakefile 464 testDict=srcDict[testname] 465 rpath=self.srcrelpath(root) 466 runscript_dir=os.path.join(self.testroot_dir,rpath) 467 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 468 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 469 470 # Get variables to go into shell scripts. last time testDict used 471 subst=self.getSubstVars(testDict,rpath,testname) 472 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 473 #if '33_' in testname: print subst['subargs'] 474 475 #Handle runfiles 476 for lfile in subst.get('localrunfiles','').split(): 477 fullfile=os.path.join(root,lfile) 478 shutil.copy(fullfile,runscript_dir) 479 # Check subtests for local runfiles 480 for stest in subst.get("subtests",[]): 481 for lfile in testDict[stest].get('localrunfiles','').split(): 482 fullfile=os.path.join(root,lfile) 483 shutil.copy(fullfile,self.runscript_dir) 484 485 # Now substitute the key variables into the header and footer 486 header=self._substVars(subst,example_template.header) 487 # The header is done twice to enable @...@ in header 488 header=self._substVars(subst,header) 489 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 490 491 # Start writing the file 492 fh.write(header+"\n") 493 494 # If there is a TODO or a SKIP then we do it before writing out the 495 # rest of the command (which is useful for working on the test) 496 # SKIP and TODO can be for the source file or for the runs 497 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 498 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 499 500 j=0 # for indentation 501 502 if loopVars: 503 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 504 if (loopHead): fh.write(loopHead+"\n") 505 506 # Subtests are special 507 if 'subtests' in testDict: 508 substP=subst # Subtests can inherit args but be careful 509 k=0 # for label suffixes 510 for stest in testDict["subtests"]: 511 subst=substP.copy() 512 subst.update(testDict[stest]) 513 # nsize is special because it is usually overwritten 514 if 'nsize' in testDict[stest]: 515 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 516 else: 517 fh.write("nsize=1\n") 518 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 519 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 520 #if '10_9' in testname: print sLoopVars 521 if sLoopVars: 522 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 523 fh.write(sLoopHead+"\n") 524 fh.write(self.getCmds(subst,j)+"\n") 525 if sLoopVars: 526 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 527 fh.write(sLoopFoot+"\n") 528 else: 529 fh.write(self.getCmds(subst,j)+"\n") 530 531 if loopVars: 532 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 533 fh.write(loopFoot+"\n") 534 535 fh.write(footer+"\n") 536 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 537 #if '10_9' in testname: sys.exit() 538 return 539 540 def genScriptsAndInfo(self,exfile,root,srcDict): 541 """ 542 Generate scripts from the source file, determine if built, etc. 543 For every test in the exfile with info in the srcDict: 544 1. Determine if it needs to be run for this arch 545 2. Generate the script 546 3. Generate the data needed to write out the makefile in a 547 convenient way 548 All tests are *always* run, but some may be SKIP'd per the TAP standard 549 """ 550 debug=False 551 execname=self.getExecname(exfile,root) 552 isBuilt=self._isBuilt(exfile,srcDict) 553 for test in srcDict: 554 if test in self.buildkeys: continue 555 if debug: print self.nameSpace(exfile,root), test 556 srcDict[test]['execname']=execname # Convenience in generating scripts 557 isRun=self._isRun(srcDict[test]) 558 self.genRunScript(test,root,isRun,srcDict) 559 srcDict[test]['isrun']=isRun 560 self.addToTests(test,root,exfile,execname,srcDict[test]) 561 562 # This adds to datastructure for building deps 563 if isBuilt: self.addToSources(exfile,root,srcDict) 564 return 565 566 def _isBuilt(self,exfile,srcDict): 567 """ 568 Determine if this file should be built. 569 """ 570 # Get the language based on file extension 571 srcDict['SKIP'] = [] 572 lang=self.getLanguage(exfile) 573 if (lang=="F" or lang=="F90"): 574 if not self.have_fortran: 575 srcDict["SKIP"].append("Fortran required for this test") 576 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 577 srcDict["SKIP"].append("Fortran f90freeform required for this test") 578 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 579 srcDict["SKIP"].append("CUDA required for this test") 580 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 581 srcDict["SKIP"].append("C++ required for this test") 582 583 # Deprecated source files 584 if srcDict.get("TODO"): 585 return False 586 587 # isRun can work with srcDict to handle the requires 588 if "requires" in srcDict: 589 if srcDict["requires"]: 590 return self._isRun(srcDict) 591 592 return srcDict['SKIP'] == [] 593 594 595 def _isRun(self,testDict): 596 """ 597 Based on the requirements listed in the src file and the petscconf.h 598 info, determine whether this test should be run or not. 599 """ 600 indent=" " 601 debug=False 602 603 if 'SKIP' not in testDict: 604 testDict['SKIP'] = [] 605 # MPI requirements 606 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 607 if debug: print indent+"Cannot run parallel tests" 608 testDict['SKIP'].append("Parallel test with serial build") 609 610 # The requirements for the test are the sum of all the run subtests 611 if 'subtests' in testDict: 612 if 'requires' not in testDict: testDict['requires']="" 613 for stest in testDict['subtests']: 614 if 'requires' in testDict[stest]: 615 testDict['requires']+=" "+testDict[stest]['requires'] 616 617 618 # Now go through all requirements 619 if 'requires' in testDict: 620 for requirement in testDict['requires'].split(): 621 requirement=requirement.strip() 622 if not requirement: continue 623 if debug: print indent+"Requirement: ", requirement 624 isNull=False 625 if requirement.startswith("!"): 626 requirement=requirement[1:]; isNull=True 627 # Precision requirement for reals 628 if requirement in self.precision_types: 629 if self.conf['PETSC_PRECISION']==requirement: 630 if isNull: 631 testDict['SKIP'].append("not "+requirement+" required") 632 continue 633 continue # Success 634 elif not isNull: 635 testDict['SKIP'].append(requirement+" required") 636 continue 637 # Precision requirement for ints 638 if requirement in self.integer_types: 639 if requirement=="int32": 640 if self.conf['PETSC_SIZEOF_INT']==4: 641 if isNull: 642 testDict['SKIP'].append("not int32 required") 643 continue 644 continue # Success 645 elif not isNull: 646 testDict['SKIP'].append("int32 required") 647 continue 648 if requirement=="int64": 649 if self.conf['PETSC_SIZEOF_INT']==8: 650 if isNull: 651 testDict['SKIP'].append("NOT int64 required") 652 continue 653 continue # Success 654 elif not isNull: 655 testDict['SKIP'].append("int64 required") 656 continue 657 # Datafilespath 658 if requirement=="datafilespath" and not isNull: 659 testDict['SKIP'].append("Requires DATAFILESPATH") 660 continue 661 # Defines -- not sure I have comments matching 662 if "define(" in requirement.lower(): 663 reqdef=requirement.split("(")[1].split(")")[0] 664 if reqdef in self.conf: 665 if isNull: 666 testDict['SKIP'].append("Null requirement not met: "+requirement) 667 continue 668 continue # Success 669 elif not isNull: 670 testDict['SKIP'].append("Required: "+requirement) 671 continue 672 673 # Rest should be packages that we can just get from conf 674 if requirement == "complex": 675 petscconfvar="PETSC_USE_COMPLEX" 676 else: 677 petscconfvar="PETSC_HAVE_"+requirement.upper() 678 if self.conf.get(petscconfvar): 679 if isNull: 680 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 681 continue 682 continue # Success 683 elif not isNull: 684 if debug: print "requirement not found: ", requirement 685 testDict['SKIP'].append(petscconfvar+" requirement not met") 686 continue 687 688 return testDict['SKIP'] == [] 689 690 def genPetscTests_summarize(self,dataDict): 691 """ 692 Required method to state what happened 693 """ 694 if not self.summarize: return 695 indent=" " 696 print(self.testroot_dir, os.path.realpath(os.curdir)) 697 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 698 fh=open(fhname,"w") 699 #print "See ", fhname 700 for root in dataDict: 701 relroot=self.srcrelpath(root) 702 pkg=relroot.split("/")[1] 703 fh.write(relroot+"\n") 704 allSrcs=[] 705 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 706 for exfile in dataDict[root]: 707 # Basic information 708 rfile=os.path.join(relroot,exfile) 709 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 710 fh.write(indent+exfile+indent*4+builtStatus+"\n") 711 712 for test in dataDict[root][exfile]: 713 if test in self.buildkeys: continue 714 line=indent*2+test 715 fh.write(line+"\n") 716 # Looks nice to have the keys in order 717 #for key in dataDict[root][exfile][test]: 718 for key in "isrun abstracted nsize args requires script".split(): 719 if key not in dataDict[root][exfile][test]: continue 720 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 721 fh.write(line+"\n") 722 fh.write("\n") 723 fh.write("\n") 724 fh.write("\n") 725 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 726 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 727 fh.close() 728 return 729 730 def genPetscTests(self,root,dirs,files,dataDict): 731 """ 732 Go through and parse the source files in the directory to generate 733 the examples based on the metadata contained in the source files 734 """ 735 debug=False 736 # Use examplesAnalyze to get what the makefles think are sources 737 #self.examplesAnalyze(root,dirs,files,anlzDict) 738 739 dataDict[root]={} 740 741 for exfile in files: 742 #TST: Until we replace files, still leaving the orginals as is 743 #if not exfile.startswith("new_"+"ex"): continue 744 #if not exfile.startswith("ex"): continue 745 746 # Ignore emacs files 747 if exfile.startswith("#") or exfile.startswith(".#"): continue 748 749 # Convenience 750 fullex=os.path.join(root,exfile) 751 if self.verbose: print(' --> '+fullex) 752 dataDict[root].update(testparse.parseTestFile(fullex,0)) 753 if exfile in dataDict[root]: 754 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 755 756 return 757 758 def walktree(self,top): 759 """ 760 Walk a directory tree, starting from 'top' 761 """ 762 #print "action", action 763 # Goal of action is to fill this dictionary 764 dataDict={} 765 for root, dirs, files in os.walk(top, topdown=False): 766 if not "examples" in root: continue 767 if not os.path.isfile(os.path.join(root,"makefile")): continue 768 if self.verbose: print(root) 769 bname=os.path.basename(root.rstrip("/")) 770 self.genPetscTests(root,dirs,files,dataDict) 771 # Now summarize this dictionary 772 self.genPetscTests_summarize(dataDict) 773 return dataDict 774 775 def gen_gnumake(self, fd): 776 """ 777 Overwrite of the method in the base PETSc class 778 """ 779 def write(stem, srcs): 780 for lang in LANGS: 781 if srcs[lang]['srcs']: 782 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 783 for pkg in PKGS: 784 srcs = self.gen_pkg(pkg) 785 write('testsrcs-' + pkg, srcs) 786 # Handle dependencies 787 for lang in LANGS: 788 for exfile in srcs[lang]['srcs']: 789 if exfile in srcs[lang]: 790 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 791 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 792 for dep in srcs[lang][exfile]: 793 fd.write(exfile+": $(TESTDIR)/"+ dep+'\n') 794 fd.write(ex +": "+exfo+" $(TESTDIR)/"+dep +'\n') 795 796 return self.gendeps 797 798 def gen_pkg(self, pkg): 799 """ 800 Overwrite of the method in the base PETSc class 801 """ 802 return self.sources[pkg] 803 804 def write_gnumake(self,dataDict): 805 """ 806 Write out something similar to files from gmakegen.py 807 808 Test depends on script which also depends on source 809 file, but since I don't have a good way generating 810 acting on a single file (oops) just depend on 811 executable which in turn will depend on src file 812 """ 813 # Different options for how to set up the targets 814 compileExecsFirst=False 815 816 # Open file 817 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 818 fd = open(arch_files, 'w') 819 820 # Write out the sources 821 gendeps = self.gen_gnumake(fd) 822 823 # Write out the tests and execname targets 824 fd.write("\n#Tests and executables\n") # Delimiter 825 826 for pkg in PKGS: 827 # These grab the ones that are built 828 for lang in LANGS: 829 testdeps=[] 830 for ftest in self.tests[pkg][lang]: 831 test=os.path.basename(ftest) 832 basedir=os.path.dirname(ftest) 833 testdeps.append(self.nameSpace(test,basedir)) 834 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 835 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 836 837 # test targets 838 for ftest in self.tests[pkg][lang]: 839 test=os.path.basename(ftest) 840 basedir=os.path.dirname(ftest) 841 testdir="${TESTDIR}/"+basedir+"/" 842 nmtest=self.nameSpace(test,basedir) 843 rundir=os.path.join(testdir,test) 844 #print test, nmtest 845 script=test+".sh" 846 847 # Deps 848 exfile=self.tests[pkg][lang][ftest]['exfile'] 849 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 850 localexec=self.tests[pkg][lang][ftest]['exec'] 851 execname=os.path.join(testdir,localexec) 852 fullscript=os.path.join(testdir,script) 853 tmpfile=os.path.join(testdir,test,test+".tmp") 854 855 # *.counts depends on the script and either executable (will 856 # be run) or the example source file (SKIP or TODO) 857 fd.write('%s.counts : %s %s' 858 % (os.path.join('$(TESTDIR)/counts', nmtest), 859 fullscript, 860 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 861 ) 862 if exfile in self.sources[pkg][lang]: 863 for dep in self.sources[pkg][lang][exfile]: 864 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 865 fd.write('\n') 866 867 # Now write the args: 868 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 869 870 fd.close() 871 return 872 873 def writeHarness(self,output,dataDict): 874 """ 875 This is set up to write out multiple harness even if only gnumake 876 is supported now 877 """ 878 eval("self.write_"+output+"(dataDict)") 879 return 880 881def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 882 if output is None: 883 output = 'gnumake' 884 885 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 886 if petsc_arch: 887 if len(petsc_arch.split(os.path.sep))>1: 888 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 889 890 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 891 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 892 testdir=testdir) 893 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 894 pEx.writeHarness(output,dataDict) 895 896if __name__ == '__main__': 897 import optparse 898 parser = optparse.OptionParser() 899 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 900 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 901 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 902 parser.add_option('--output', help='Location to write output file', default=None) 903 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 904 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 905 opts, extra_args = parser.parse_args() 906 opts, extra_args = parser.parse_args() 907 if extra_args: 908 import sys 909 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 910 exit(1) 911 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 912 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 913