1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 115 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 116 nameString=prefix+srcfile 117 else: 118 #nameString=srcdir+": "+srcfile 119 nameString=srcfile 120 return nameString 121 122 def getLanguage(self,srcfile): 123 """ 124 Based on the source, determine associated language as found in gmakegen.LANGS 125 Can we just return srcext[1:\] now? 126 """ 127 langReq=None 128 srcext=os.path.splitext(srcfile)[-1] 129 if srcext in ".F90".split(): langReq="F90" 130 if srcext in ".F".split(): langReq="F" 131 if srcext in ".cxx".split(): langReq="cxx" 132 if srcext == ".cu": langReq="cu" 133 if srcext == ".c": langReq="c" 134 #if not langReq: print "ERROR: ", srcext, srcfile 135 return langReq 136 137 def _getLoopVars(self,inDict,testname, isSubtest=False): 138 """ 139 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 140 Return: 141 inDict['args']: -ksp_monitor 142 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 143 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 144 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 145 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 146 subst should be passed in instead of inDict 147 """ 148 loopVars={}; newargs="" 149 lkeys=inDict.keys() 150 lsuffix='_' 151 from testparse import parseLoopArgs 152 for key in lkeys: 153 if type(inDict[key])!=types.StringType: continue 154 keystr = str(inDict[key]) 155 akey=('subargs' if key=='args' else key) # what to assign 156 if akey not in inDict: inDict[akey]='' 157 varlist=[] 158 for varset in re.split('-(?=[a-zA-Z])',keystr): 159 if not varset.strip(): continue 160 if '{{' in varset: 161 keyvar,lvars,ftype=parseLoopArgs(varset) 162 if akey not in loopVars: loopVars[akey]={} 163 varlist.append(keyvar) 164 loopVars[akey][keyvar]=[keyvar,lvars] 165 if akey=='nsize': 166 inDict[akey] = '${' + keyvar + '}' 167 lsuffix+=akey+'-'+inDict[akey]+'_' 168 else: 169 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 170 lsuffix+=keyvar+'-${' + keyvar + '}_' 171 else: 172 if key=='args': newargs+=" -"+varset.strip() 173 if varlist: loopVars[akey]['varlist']=varlist 174 175 176 # For subtests, args are always substituted in (not top level) 177 if isSubtest: 178 inDict['subargs']+=" "+newargs.strip() 179 inDict['args']='' 180 if 'label_suffix' in inDict: 181 inDict['label_suffix']+=lsuffix.rstrip('_') 182 else: 183 inDict['label_suffix']=lsuffix.rstrip('_') 184 else: 185 if loopVars.keys(): 186 inDict['args']=newargs.strip() 187 inDict['label_suffix']=lsuffix.rstrip('_') 188 if loopVars.keys(): 189 return loopVars 190 else: 191 return None 192 193 def getArgLabel(self,testDict): 194 """ 195 In all of the arguments in the test dictionary, create a simple 196 string for searching within the makefile system. For simplicity in 197 search, remove "-", for strings, etc. 198 Also, concatenate the arg commands 199 For now, ignore nsize -- seems hard to search for anyway 200 """ 201 # Collect all of the args associated with a test 202 argStr=("" if 'args' not in testDict else testDict['args']) 203 if 'subtests' in testDict: 204 for stest in testDict["subtests"]: 205 sd=testDict[stest] 206 argStr=argStr+("" if 'args' not in sd else sd['args']) 207 208 # Now go through and cleanup 209 argStr=re.sub('{{(.*?)}}',"",argStr) 210 argStr=re.sub('-'," ",argStr) 211 for digit in string.digits: argStr=re.sub(digit," ",argStr) 212 argStr=re.sub("\.","",argStr) 213 argStr=re.sub(",","",argStr) 214 argStr=re.sub('\+',' ',argStr) 215 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 216 return argStr.strip() 217 218 def addToSources(self,exfile,root,srcDict): 219 """ 220 Put into data structure that allows easy generation of makefile 221 """ 222 rpath=self.srcrelpath(root) 223 pkg=rpath.split(os.path.sep)[1] 224 relpfile=os.path.join(rpath,exfile) 225 lang=self.getLanguage(exfile) 226 if not lang: return 227 self.sources[pkg][lang]['srcs'].append(relpfile) 228 if 'depends' in srcDict: 229 depSrc=srcDict['depends'] 230 depObj=os.path.splitext(depSrc)[0]+".o" 231 self.sources[pkg][lang][relpfile]=os.path.join(rpath,depObj) 232 233 # In gmakefile, ${TESTDIR} var specifies the object compilation 234 testsdir=self.srcrelpath(root)+"/" 235 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 236 self.objects[pkg].append(objfile) 237 return 238 239 def addToTests(self,test,root,exfile,execname,testDict): 240 """ 241 Put into data structure that allows easy generation of makefile 242 Organized by languages to allow testing of languages 243 """ 244 rpath=self.srcrelpath(root) 245 pkg=rpath.split("/")[1] 246 #nmtest=self.nameSpace(test,root) 247 nmtest=os.path.join(rpath,test) 248 lang=self.getLanguage(exfile) 249 if not lang: return 250 self.tests[pkg][lang][nmtest]={} 251 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 252 self.tests[pkg][lang][nmtest]['exec']=execname 253 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 254 return 255 256 def getExecname(self,exfile,root): 257 """ 258 Generate bash script using template found next to this file. 259 This file is read in at constructor time to avoid file I/O 260 """ 261 rpath=self.srcrelpath(root) 262 if self.single_ex: 263 execname=rpath.split("/")[1]+"-ex" 264 else: 265 execname=os.path.splitext(exfile)[0] 266 return execname 267 268 def getSubstVars(self,testDict,rpath,testname): 269 """ 270 Create a dictionary with all of the variables that get substituted 271 into the template commands found in example_template.py 272 """ 273 subst={} 274 275 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 276 if 'nsize' not in testDict: testDict['nsize']=1 277 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 278 for ak in testparse.acceptedkeys: 279 if ak=='test': continue 280 subst[ak]=(testDict[ak] if ak in testDict else '') 281 282 # Now do other variables 283 subst['execname']=testDict['execname'] 284 if 'filter' in testDict: 285 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 286 287 # Others 288 subst['subargs']='' # Default. For variables override 289 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 290 subst['label_suffix']='' 291 subst['comments']="\n#".join(subst['comments'].split("\n")) 292 if subst['comments']: subst['comments']="#"+subst['comments'] 293 subst['exec']="../"+subst['execname'] 294 subst['testroot']=self.testroot_dir 295 subst['testname']=testname 296 dp = self.conf.get('DATAFILESPATH','') 297 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 298 299 # This is used to label some matrices 300 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 301 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 302 303 # These can have for loops and are treated separately later 304 subst['nsize']=str(subst['nsize']) 305 306 #Conf vars 307 if self.petsc_arch.find('valgrind')>=0: 308 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 309 else: 310 subst['mpiexec']=self.conf['MPIEXEC'] 311 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 312 subst['petsc_arch']=self.petsc_arch 313 if not self.inInstallDir: 314 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 315 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 316 else: 317 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 318 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 319 subst['diff']=self.conf['DIFF'] 320 subst['rm']=self.conf['RM'] 321 subst['grep']=self.conf['GREP'] 322 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 323 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 324 325 # Output file is special because of subtests override 326 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 327 if not "_" in defroot: defroot=defroot+"_1" 328 subst['defroot']=defroot 329 subst['label']=self.nameSpace(defroot,subst['srcdir']) 330 subst['redirect_file']=defroot+".tmp" 331 if 'output_file' not in testDict: 332 subst['output_file']="output/"+defroot+".out" 333 # Add in the full path here. 334 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 335 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 336 if not subst['TODO']: 337 print "Warning: "+subst['output_file']+" not found." 338 # Worry about alt files here -- see 339 # src/snes/examples/tutorials/output/ex22*.out 340 altlist=[subst['output_file']] 341 for i in range(1,3): 342 altroot=defroot+"_alt" 343 if i==2: altroot=altroot+"_2" 344 af="output/"+altroot+".out" 345 srcaf=os.path.join(subst['srcdir'],af) 346 fullaf=os.path.join(self.petsc_dir,srcaf) 347 if os.path.isfile(fullaf): altlist.append(srcaf) 348 if len(altlist)>1: subst['altfiles']=altlist 349 #if len(altlist)>1: print "Found alt files: ",altlist 350 351 return subst 352 353 def getCmds(self,subst,i): 354 """ 355 Generate bash script using template found next to this file. 356 This file is read in at constructor time to avoid file I/O 357 """ 358 nindnt=i # the start and has to be consistent with below 359 cmdindnt=self.indent*nindnt 360 cmdLines="" 361 362 # MPI is the default -- but we have a few odd commands 363 if not subst['command']: 364 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 365 else: 366 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 367 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 368 369 cmdLines+=cmdindnt+'if test $res = 0; then\n' 370 diffindnt=self.indent*(nindnt+1) 371 if not subst['filter_output']: 372 if 'altfiles' not in subst: 373 cmd=diffindnt+self._substVars(subst,example_template.difftest) 374 else: 375 # Have to do it by hand a bit because of variable number of alt files 376 rf=subst['redirect_file'] 377 cmd=diffindnt+example_template.difftest.split('@')[0] 378 for i in range(len(subst['altfiles'])): 379 af=subst['altfiles'][i] 380 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 381 if i!=len(subst['altfiles'])-1: 382 cmd+=' || ${diff_exe} ' 383 else: 384 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 385 cmd+=subst['label_suffix']+' ""' # Quotes are painful 386 else: 387 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 388 cmdLines+=cmd+"\n" 389 cmdLines+=cmdindnt+'else\n' 390 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 391 cmdLines+=cmdindnt+'fi\n' 392 return cmdLines 393 394 def _substVars(self,subst,origStr): 395 """ 396 Substitute variables 397 """ 398 Str=origStr 399 for subkey in subst: 400 if type(subst[subkey])!=types.StringType: continue 401 patt="@"+subkey.upper()+"@" 402 Str=re.sub(patt,subst[subkey],Str) 403 return Str 404 405 def _writeTodoSkip(self,fh,tors,reasons,footer): 406 """ 407 Write out the TODO and SKIP lines in the file 408 The TODO or SKIP variable, tors, should be lower case 409 """ 410 TORS=tors.upper() 411 template=eval("example_template."+tors+"line") 412 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 413 tab = '' 414 if reasons: 415 fh.write('if ! $force; then\n') 416 tab = tab + ' ' 417 if reasons == ["Requires DATAFILESPATH"]: 418 # The only reason not to run is DATAFILESPATH, which we check at run-time 419 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 420 tab = tab + ' ' 421 if reasons: 422 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 423 fh.write(tab+footer+"\n") 424 fh.write(tab+"exit\n") 425 if reasons == ["Requires DATAFILESPATH"]: 426 fh.write(' fi\n') 427 if reasons: 428 fh.write('fi\n') 429 fh.write('\n\n') 430 return 431 432 def getLoopVarsHead(self,loopVars,i): 433 """ 434 Generate a nicely indented string with the format loops 435 Here is what the data structure looks like 436 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 437 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 438 loopVars['subargs']['pc_type']=["j","cholesky sor"] 439 """ 440 outstr=''; indnt=self.indent 441 for key in loopVars: 442 for var in loopVars[key]['varlist']: 443 varval=loopVars[key][var] 444 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 445 i = i + 1 446 return (outstr,i) 447 448 def getLoopVarsFoot(self,loopVars,i): 449 outstr=''; indnt=self.indent 450 for key in loopVars: 451 for var in loopVars[key]['varlist']: 452 i = i - 1 453 outstr += indnt * i + "done\n" 454 return (outstr,i) 455 456 def genRunScript(self,testname,root,isRun,srcDict): 457 """ 458 Generate bash script using template found next to this file. 459 This file is read in at constructor time to avoid file I/O 460 """ 461 # runscript_dir directory has to be consistent with gmakefile 462 testDict=srcDict[testname] 463 rpath=self.srcrelpath(root) 464 runscript_dir=os.path.join(self.testroot_dir,rpath) 465 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 466 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 467 468 # Get variables to go into shell scripts. last time testDict used 469 subst=self.getSubstVars(testDict,rpath,testname) 470 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 471 #if '33_' in testname: print subst['subargs'] 472 473 #Handle runfiles 474 for lfile in subst.get('localrunfiles','').split(): 475 fullfile=os.path.join(root,lfile) 476 shutil.copy(fullfile,runscript_dir) 477 # Check subtests for local runfiles 478 for stest in subst.get("subtests",[]): 479 for lfile in testDict[stest].get('localrunfiles','').split(): 480 fullfile=os.path.join(root,lfile) 481 shutil.copy(fullfile,self.runscript_dir) 482 483 # Now substitute the key variables into the header and footer 484 header=self._substVars(subst,example_template.header) 485 # The header is done twice to enable @...@ in header 486 header=self._substVars(subst,header) 487 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 488 489 # Start writing the file 490 fh.write(header+"\n") 491 492 # If there is a TODO or a SKIP then we do it before writing out the 493 # rest of the command (which is useful for working on the test) 494 # SKIP and TODO can be for the source file or for the runs 495 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 496 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 497 498 j=0 # for indentation 499 500 if loopVars: 501 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 502 if (loopHead): fh.write(loopHead+"\n") 503 504 # Subtests are special 505 if 'subtests' in testDict: 506 substP=subst # Subtests can inherit args but be careful 507 k=0 # for label suffixes 508 for stest in testDict["subtests"]: 509 subst=substP.copy() 510 subst.update(testDict[stest]) 511 # nsize is special because it is usually overwritten 512 if 'nsize' in testDict[stest]: 513 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 514 else: 515 fh.write("nsize=1\n") 516 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 517 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 518 #if '10_9' in testname: print sLoopVars 519 if sLoopVars: 520 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 521 fh.write(sLoopHead+"\n") 522 fh.write(self.getCmds(subst,j)+"\n") 523 if sLoopVars: 524 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 525 fh.write(sLoopFoot+"\n") 526 else: 527 fh.write(self.getCmds(subst,j)+"\n") 528 529 if loopVars: 530 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 531 fh.write(loopFoot+"\n") 532 533 fh.write(footer+"\n") 534 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 535 #if '10_9' in testname: sys.exit() 536 return 537 538 def genScriptsAndInfo(self,exfile,root,srcDict): 539 """ 540 Generate scripts from the source file, determine if built, etc. 541 For every test in the exfile with info in the srcDict: 542 1. Determine if it needs to be run for this arch 543 2. Generate the script 544 3. Generate the data needed to write out the makefile in a 545 convenient way 546 All tests are *always* run, but some may be SKIP'd per the TAP standard 547 """ 548 debug=False 549 execname=self.getExecname(exfile,root) 550 isBuilt=self._isBuilt(exfile,srcDict) 551 for test in srcDict: 552 if test in self.buildkeys: continue 553 if debug: print self.nameSpace(exfile,root), test 554 srcDict[test]['execname']=execname # Convenience in generating scripts 555 isRun=self._isRun(srcDict[test]) 556 self.genRunScript(test,root,isRun,srcDict) 557 srcDict[test]['isrun']=isRun 558 self.addToTests(test,root,exfile,execname,srcDict[test]) 559 560 # This adds to datastructure for building deps 561 if isBuilt: self.addToSources(exfile,root,srcDict) 562 return 563 564 def _isBuilt(self,exfile,srcDict): 565 """ 566 Determine if this file should be built. 567 """ 568 # Get the language based on file extension 569 srcDict['SKIP'] = [] 570 lang=self.getLanguage(exfile) 571 if (lang=="F" or lang=="F90"): 572 if not self.have_fortran: 573 srcDict["SKIP"].append("Fortran required for this test") 574 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 575 srcDict["SKIP"].append("Fortran f90freeform required for this test") 576 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 577 srcDict["SKIP"].append("CUDA required for this test") 578 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 579 srcDict["SKIP"].append("C++ required for this test") 580 581 # Deprecated source files 582 if srcDict.get("TODO"): 583 return False 584 585 # isRun can work with srcDict to handle the requires 586 if "requires" in srcDict: 587 if srcDict["requires"]: 588 return self._isRun(srcDict) 589 590 return srcDict['SKIP'] == [] 591 592 593 def _isRun(self,testDict): 594 """ 595 Based on the requirements listed in the src file and the petscconf.h 596 info, determine whether this test should be run or not. 597 """ 598 indent=" " 599 debug=False 600 601 if 'SKIP' not in testDict: 602 testDict['SKIP'] = [] 603 # MPI requirements 604 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 605 if debug: print indent+"Cannot run parallel tests" 606 testDict['SKIP'].append("Parallel test with serial build") 607 608 # The requirements for the test are the sum of all the run subtests 609 if 'subtests' in testDict: 610 if 'requires' not in testDict: testDict['requires']="" 611 for stest in testDict['subtests']: 612 if 'requires' in testDict[stest]: 613 testDict['requires']+=" "+testDict[stest]['requires'] 614 615 616 # Now go through all requirements 617 if 'requires' in testDict: 618 for requirement in testDict['requires'].split(): 619 requirement=requirement.strip() 620 if not requirement: continue 621 if debug: print indent+"Requirement: ", requirement 622 isNull=False 623 if requirement.startswith("!"): 624 requirement=requirement[1:]; isNull=True 625 # Precision requirement for reals 626 if requirement in self.precision_types: 627 if self.conf['PETSC_PRECISION']==requirement: 628 if isNull: 629 testDict['SKIP'].append("not "+requirement+" required") 630 continue 631 continue # Success 632 elif not isNull: 633 testDict['SKIP'].append(requirement+" required") 634 continue 635 # Precision requirement for ints 636 if requirement in self.integer_types: 637 if requirement=="int32": 638 if self.conf['PETSC_SIZEOF_INT']==4: 639 if isNull: 640 testDict['SKIP'].append("not int32 required") 641 continue 642 continue # Success 643 elif not isNull: 644 testDict['SKIP'].append("int32 required") 645 continue 646 if requirement=="int64": 647 if self.conf['PETSC_SIZEOF_INT']==8: 648 if isNull: 649 testDict['SKIP'].append("NOT int64 required") 650 continue 651 continue # Success 652 elif not isNull: 653 testDict['SKIP'].append("int64 required") 654 continue 655 # Datafilespath 656 if requirement=="datafilespath" and not isNull: 657 testDict['SKIP'].append("Requires DATAFILESPATH") 658 continue 659 # Defines -- not sure I have comments matching 660 if "define(" in requirement.lower(): 661 reqdef=requirement.split("(")[1].split(")")[0] 662 if reqdef in self.conf: 663 if isNull: 664 testDict['SKIP'].append("Null requirement not met: "+requirement) 665 continue 666 continue # Success 667 elif not isNull: 668 testDict['SKIP'].append("Required: "+requirement) 669 continue 670 671 # Rest should be packages that we can just get from conf 672 if requirement == "complex": 673 petscconfvar="PETSC_USE_COMPLEX" 674 else: 675 petscconfvar="PETSC_HAVE_"+requirement.upper() 676 if self.conf.get(petscconfvar): 677 if isNull: 678 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 679 continue 680 continue # Success 681 elif not isNull: 682 if debug: print "requirement not found: ", requirement 683 testDict['SKIP'].append(petscconfvar+" requirement not met") 684 continue 685 686 return testDict['SKIP'] == [] 687 688 def genPetscTests_summarize(self,dataDict): 689 """ 690 Required method to state what happened 691 """ 692 if not self.summarize: return 693 indent=" " 694 print(self.testroot_dir, os.path.realpath(os.curdir)) 695 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 696 fh=open(fhname,"w") 697 #print "See ", fhname 698 for root in dataDict: 699 relroot=self.srcrelpath(root) 700 pkg=relroot.split("/")[1] 701 fh.write(relroot+"\n") 702 allSrcs=[] 703 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 704 for exfile in dataDict[root]: 705 # Basic information 706 rfile=os.path.join(relroot,exfile) 707 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 708 fh.write(indent+exfile+indent*4+builtStatus+"\n") 709 710 for test in dataDict[root][exfile]: 711 if test in self.buildkeys: continue 712 line=indent*2+test 713 fh.write(line+"\n") 714 # Looks nice to have the keys in order 715 #for key in dataDict[root][exfile][test]: 716 for key in "isrun abstracted nsize args requires script".split(): 717 if key not in dataDict[root][exfile][test]: continue 718 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 719 fh.write(line+"\n") 720 fh.write("\n") 721 fh.write("\n") 722 fh.write("\n") 723 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 724 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 725 fh.close() 726 return 727 728 def genPetscTests(self,root,dirs,files,dataDict): 729 """ 730 Go through and parse the source files in the directory to generate 731 the examples based on the metadata contained in the source files 732 """ 733 debug=False 734 # Use examplesAnalyze to get what the makefles think are sources 735 #self.examplesAnalyze(root,dirs,files,anlzDict) 736 737 dataDict[root]={} 738 739 for exfile in files: 740 #TST: Until we replace files, still leaving the orginals as is 741 #if not exfile.startswith("new_"+"ex"): continue 742 #if not exfile.startswith("ex"): continue 743 744 # Ignore emacs files 745 if exfile.startswith("#") or exfile.startswith(".#"): continue 746 747 # Convenience 748 fullex=os.path.join(root,exfile) 749 if self.verbose: print(' --> '+fullex) 750 dataDict[root].update(testparse.parseTestFile(fullex,0)) 751 if exfile in dataDict[root]: 752 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 753 754 return 755 756 def walktree(self,top): 757 """ 758 Walk a directory tree, starting from 'top' 759 """ 760 #print "action", action 761 # Goal of action is to fill this dictionary 762 dataDict={} 763 for root, dirs, files in os.walk(top, topdown=False): 764 if not "examples" in root: continue 765 if not os.path.isfile(os.path.join(root,"makefile")): continue 766 if self.verbose: print(root) 767 bname=os.path.basename(root.rstrip("/")) 768 self.genPetscTests(root,dirs,files,dataDict) 769 # Now summarize this dictionary 770 self.genPetscTests_summarize(dataDict) 771 return dataDict 772 773 def gen_gnumake(self, fd): 774 """ 775 Overwrite of the method in the base PETSc class 776 """ 777 def write(stem, srcs): 778 for lang in LANGS: 779 if srcs[lang]['srcs']: 780 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 781 for pkg in PKGS: 782 srcs = self.gen_pkg(pkg) 783 write('testsrcs-' + pkg, srcs) 784 # Handle dependencies 785 for lang in LANGS: 786 for exfile in srcs[lang]['srcs']: 787 if exfile in srcs[lang]: 788 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 789 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 790 fd.write(exfile+": $(TESTDIR)/"+ srcs[lang][exfile]+'\n') 791 fd.write(ex +": "+exfo+" $(TESTDIR)/"+srcs[lang][exfile] +'\n') 792 793 return self.gendeps 794 795 def gen_pkg(self, pkg): 796 """ 797 Overwrite of the method in the base PETSc class 798 """ 799 return self.sources[pkg] 800 801 def write_gnumake(self,dataDict): 802 """ 803 Write out something similar to files from gmakegen.py 804 805 Test depends on script which also depends on source 806 file, but since I don't have a good way generating 807 acting on a single file (oops) just depend on 808 executable which in turn will depend on src file 809 """ 810 # Different options for how to set up the targets 811 compileExecsFirst=False 812 813 # Open file 814 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 815 fd = open(arch_files, 'w') 816 817 # Write out the sources 818 gendeps = self.gen_gnumake(fd) 819 820 # Write out the tests and execname targets 821 fd.write("\n#Tests and executables\n") # Delimiter 822 823 for pkg in PKGS: 824 # These grab the ones that are built 825 for lang in LANGS: 826 testdeps=[] 827 for ftest in self.tests[pkg][lang]: 828 test=os.path.basename(ftest) 829 basedir=os.path.dirname(ftest) 830 testdeps.append(self.nameSpace(test,basedir)) 831 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 832 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 833 834 # test targets 835 for ftest in self.tests[pkg][lang]: 836 test=os.path.basename(ftest) 837 basedir=os.path.dirname(ftest) 838 testdir="${TESTDIR}/"+basedir+"/" 839 nmtest=self.nameSpace(test,basedir) 840 rundir=os.path.join(testdir,test) 841 #print test, nmtest 842 script=test+".sh" 843 844 # Deps 845 exfile=self.tests[pkg][lang][ftest]['exfile'] 846 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 847 localexec=self.tests[pkg][lang][ftest]['exec'] 848 execname=os.path.join(testdir,localexec) 849 fullscript=os.path.join(testdir,script) 850 tmpfile=os.path.join(testdir,test,test+".tmp") 851 852 # *.counts depends on the script and either executable (will 853 # be run) or the example source file (SKIP or TODO) 854 if exfile in self.sources[pkg][lang]: 855 fd.write('%s.counts : %s %s %s\n' 856 % (os.path.join('$(TESTDIR)/counts', nmtest), 857 fullscript, 858 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex, 859 os.path.join('$(TESTDIR)',self.sources[pkg][lang][exfile])) 860 ) 861 else: 862 fd.write('%s.counts : %s %s\n' 863 % (os.path.join('$(TESTDIR)/counts', nmtest), 864 fullscript, 865 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 866 ) 867 868 # Now write the args: 869 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 870 871 fd.close() 872 return 873 874 def writeHarness(self,output,dataDict): 875 """ 876 This is set up to write out multiple harness even if only gnumake 877 is supported now 878 """ 879 eval("self.write_"+output+"(dataDict)") 880 return 881 882def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 883 if output is None: 884 output = 'gnumake' 885 886 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 887 if petsc_arch: 888 if len(petsc_arch.split(os.path.sep))>1: 889 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 890 891 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 892 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 893 testdir=testdir) 894 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 895 pEx.writeHarness(output,dataDict) 896 897if __name__ == '__main__': 898 import optparse 899 parser = optparse.OptionParser() 900 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 901 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 902 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 903 parser.add_option('--output', help='Location to write output file', default=None) 904 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 905 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 906 opts, extra_args = parser.parse_args() 907 opts, extra_args = parser.parse_args() 908 if extra_args: 909 import sys 910 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 911 exit(1) 912 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 913 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 914