1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 115 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 116 nameString=prefix+srcfile 117 else: 118 #nameString=srcdir+": "+srcfile 119 nameString=srcfile 120 return nameString 121 122 def getLanguage(self,srcfile): 123 """ 124 Based on the source, determine associated language as found in gmakegen.LANGS 125 Can we just return srcext[1:\] now? 126 """ 127 langReq=None 128 srcext=os.path.splitext(srcfile)[-1] 129 if srcext in ".F90".split(): langReq="F90" 130 if srcext in ".F".split(): langReq="F" 131 if srcext in ".cxx".split(): langReq="cxx" 132 if srcext == ".cu": langReq="cu" 133 if srcext == ".c": langReq="c" 134 #if not langReq: print "ERROR: ", srcext, srcfile 135 return langReq 136 137 def _getLoopVars(self,inDict,testname, isSubtest=False): 138 """ 139 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 140 Return: 141 inDict['args']: -ksp_monitor 142 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 143 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 144 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 145 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 146 subst should be passed in instead of inDict 147 """ 148 loopVars={}; newargs="" 149 lkeys=inDict.keys() 150 lsuffix='_' 151 from testparse import parseLoopArgs 152 for key in lkeys: 153 if type(inDict[key])!=types.StringType: continue 154 keystr = str(inDict[key]) 155 akey=('subargs' if key=='args' else key) # what to assign 156 if akey not in inDict: inDict[akey]='' 157 varlist=[] 158 for varset in re.split('-(?=[a-zA-Z])',keystr): 159 if not varset.strip(): continue 160 if '{{' in varset: 161 keyvar,lvars,ftype=parseLoopArgs(varset) 162 if akey not in loopVars: loopVars[akey]={} 163 varlist.append(keyvar) 164 loopVars[akey][keyvar]=[keyvar,lvars] 165 if akey=='nsize': 166 inDict[akey] = '${' + keyvar + '}' 167 lsuffix+=akey+'-'+inDict[akey]+'_' 168 else: 169 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 170 lsuffix+=keyvar+'-${' + keyvar + '}_' 171 else: 172 if key=='args': newargs+=" -"+varset.strip() 173 if varlist: loopVars[akey]['varlist']=varlist 174 175 176 # For subtests, args are always substituted in (not top level) 177 if isSubtest: 178 inDict['subargs']+=" "+newargs.strip() 179 inDict['args']='' 180 if 'label_suffix' in inDict: 181 inDict['label_suffix']+=lsuffix.rstrip('_') 182 else: 183 inDict['label_suffix']=lsuffix.rstrip('_') 184 else: 185 if loopVars.keys(): 186 inDict['args']=newargs.strip() 187 inDict['label_suffix']=lsuffix.rstrip('_') 188 if loopVars.keys(): 189 return loopVars 190 else: 191 return None 192 193 def getArgLabel(self,testDict): 194 """ 195 In all of the arguments in the test dictionary, create a simple 196 string for searching within the makefile system. For simplicity in 197 search, remove "-", for strings, etc. 198 Also, concatenate the arg commands 199 For now, ignore nsize -- seems hard to search for anyway 200 """ 201 # Collect all of the args associated with a test 202 argStr=("" if 'args' not in testDict else testDict['args']) 203 if 'subtests' in testDict: 204 for stest in testDict["subtests"]: 205 sd=testDict[stest] 206 argStr=argStr+("" if 'args' not in sd else sd['args']) 207 208 # Now go through and cleanup 209 argStr=re.sub('{{(.*?)}}',"",argStr) 210 argStr=re.sub('-'," ",argStr) 211 for digit in string.digits: argStr=re.sub(digit," ",argStr) 212 argStr=re.sub("\.","",argStr) 213 argStr=re.sub(",","",argStr) 214 argStr=re.sub('\+',' ',argStr) 215 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 216 return argStr.strip() 217 218 def addToSources(self,exfile,root,srcDict): 219 """ 220 Put into data structure that allows easy generation of makefile 221 """ 222 rpath=self.srcrelpath(root) 223 pkg=rpath.split(os.path.sep)[1] 224 relpfile=os.path.join(rpath,exfile) 225 lang=self.getLanguage(exfile) 226 if not lang: return 227 self.sources[pkg][lang]['srcs'].append(relpfile) 228 self.sources[pkg][lang][relpfile] = [] 229 if 'depends' in srcDict: 230 depSrcList=srcDict['depends'].split() 231 for depSrc in depSrcList: 232 depObj=os.path.splitext(depSrc)[0]+".o" 233 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 234 235 # In gmakefile, ${TESTDIR} var specifies the object compilation 236 testsdir=self.srcrelpath(root)+"/" 237 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 238 self.objects[pkg].append(objfile) 239 return 240 241 def addToTests(self,test,root,exfile,execname,testDict): 242 """ 243 Put into data structure that allows easy generation of makefile 244 Organized by languages to allow testing of languages 245 """ 246 rpath=self.srcrelpath(root) 247 pkg=rpath.split("/")[1] 248 #nmtest=self.nameSpace(test,root) 249 nmtest=os.path.join(rpath,test) 250 lang=self.getLanguage(exfile) 251 if not lang: return 252 self.tests[pkg][lang][nmtest]={} 253 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 254 self.tests[pkg][lang][nmtest]['exec']=execname 255 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 256 return 257 258 def getExecname(self,exfile,root): 259 """ 260 Generate bash script using template found next to this file. 261 This file is read in at constructor time to avoid file I/O 262 """ 263 rpath=self.srcrelpath(root) 264 if self.single_ex: 265 execname=rpath.split("/")[1]+"-ex" 266 else: 267 execname=os.path.splitext(exfile)[0] 268 return execname 269 270 def getSubstVars(self,testDict,rpath,testname): 271 """ 272 Create a dictionary with all of the variables that get substituted 273 into the template commands found in example_template.py 274 """ 275 subst={} 276 277 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 278 if 'nsize' not in testDict: testDict['nsize']=1 279 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 280 for ak in testparse.acceptedkeys: 281 if ak=='test': continue 282 subst[ak]=(testDict[ak] if ak in testDict else '') 283 284 # Now do other variables 285 subst['execname']=testDict['execname'] 286 if 'filter' in testDict: 287 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 288 289 # Others 290 subst['subargs']='' # Default. For variables override 291 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 292 subst['label_suffix']='' 293 subst['comments']="\n#".join(subst['comments'].split("\n")) 294 if subst['comments']: subst['comments']="#"+subst['comments'] 295 subst['exec']="../"+subst['execname'] 296 subst['testroot']=self.testroot_dir 297 subst['testname']=testname 298 dp = self.conf.get('DATAFILESPATH','') 299 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 300 301 # This is used to label some matrices 302 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 303 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 304 305 # These can have for loops and are treated separately later 306 subst['nsize']=str(subst['nsize']) 307 308 #Conf vars 309 if self.petsc_arch.find('valgrind')>=0: 310 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 311 else: 312 subst['mpiexec']=self.conf['MPIEXEC'] 313 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 314 subst['petsc_arch']=self.petsc_arch 315 if not self.inInstallDir: 316 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 317 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 318 else: 319 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 320 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 321 subst['diff']=self.conf['DIFF'] 322 subst['rm']=self.conf['RM'] 323 subst['grep']=self.conf['GREP'] 324 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 325 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 326 327 # Output file is special because of subtests override 328 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 329 if not "_" in defroot: defroot=defroot+"_1" 330 subst['defroot']=defroot 331 subst['label']=self.nameSpace(defroot,subst['srcdir']) 332 subst['redirect_file']=defroot+".tmp" 333 if 'output_file' not in testDict: 334 subst['output_file']="output/"+defroot+".out" 335 # Add in the full path here. 336 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 337 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 338 if not subst['TODO']: 339 print "Warning: "+subst['output_file']+" not found." 340 # Worry about alt files here -- see 341 # src/snes/examples/tutorials/output/ex22*.out 342 altlist=[subst['output_file']] 343 basefile,ext = os.path.splitext(subst['output_file']) 344 for i in range(1,9): 345 altroot=basefile+"_alt" 346 if i > 1: altroot=altroot+"_"+str(i) 347 af=altroot+".out" 348 srcaf=os.path.join(subst['srcdir'],af) 349 fullaf=os.path.join(self.petsc_dir,srcaf) 350 if os.path.isfile(fullaf): altlist.append(srcaf) 351 if len(altlist)>1: subst['altfiles']=altlist 352 #if len(altlist)>1: print "Found alt files: ",altlist 353 354 return subst 355 356 def getCmds(self,subst,i): 357 """ 358 Generate bash script using template found next to this file. 359 This file is read in at constructor time to avoid file I/O 360 """ 361 nindnt=i # the start and has to be consistent with below 362 cmdindnt=self.indent*nindnt 363 cmdLines="" 364 365 # MPI is the default -- but we have a few odd commands 366 if not subst['command']: 367 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 368 else: 369 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 370 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 371 372 cmdLines+=cmdindnt+'if test $res = 0; then\n' 373 diffindnt=self.indent*(nindnt+1) 374 if not subst['filter_output']: 375 if 'altfiles' not in subst: 376 cmd=diffindnt+self._substVars(subst,example_template.difftest) 377 else: 378 # Have to do it by hand a bit because of variable number of alt files 379 rf=subst['redirect_file'] 380 cmd=diffindnt+example_template.difftest.split('@')[0] 381 for i in range(len(subst['altfiles'])): 382 af=subst['altfiles'][i] 383 cmd+=af+' '+rf+' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 384 if i!=len(subst['altfiles'])-1: 385 cmd+=' || ${diff_exe} ' 386 else: 387 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 388 cmd+=subst['label_suffix']+' ""' # Quotes are painful 389 else: 390 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 391 cmdLines+=cmd+"\n" 392 cmdLines+=cmdindnt+'else\n' 393 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 394 cmdLines+=cmdindnt+'fi\n' 395 return cmdLines 396 397 def _substVars(self,subst,origStr): 398 """ 399 Substitute variables 400 """ 401 Str=origStr 402 for subkey in subst: 403 if type(subst[subkey])!=types.StringType: continue 404 patt="@"+subkey.upper()+"@" 405 Str=re.sub(patt,subst[subkey],Str) 406 return Str 407 408 def _writeTodoSkip(self,fh,tors,reasons,footer): 409 """ 410 Write out the TODO and SKIP lines in the file 411 The TODO or SKIP variable, tors, should be lower case 412 """ 413 TORS=tors.upper() 414 template=eval("example_template."+tors+"line") 415 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 416 tab = '' 417 if reasons: 418 fh.write('if ! $force; then\n') 419 tab = tab + ' ' 420 if reasons == ["Requires DATAFILESPATH"]: 421 # The only reason not to run is DATAFILESPATH, which we check at run-time 422 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 423 tab = tab + ' ' 424 if reasons: 425 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 426 fh.write(tab+footer+"\n") 427 fh.write(tab+"exit\n") 428 if reasons == ["Requires DATAFILESPATH"]: 429 fh.write(' fi\n') 430 if reasons: 431 fh.write('fi\n') 432 fh.write('\n\n') 433 return 434 435 def getLoopVarsHead(self,loopVars,i): 436 """ 437 Generate a nicely indented string with the format loops 438 Here is what the data structure looks like 439 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 440 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 441 loopVars['subargs']['pc_type']=["j","cholesky sor"] 442 """ 443 outstr=''; indnt=self.indent 444 for key in loopVars: 445 for var in loopVars[key]['varlist']: 446 varval=loopVars[key][var] 447 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 448 i = i + 1 449 return (outstr,i) 450 451 def getLoopVarsFoot(self,loopVars,i): 452 outstr=''; indnt=self.indent 453 for key in loopVars: 454 for var in loopVars[key]['varlist']: 455 i = i - 1 456 outstr += indnt * i + "done\n" 457 return (outstr,i) 458 459 def genRunScript(self,testname,root,isRun,srcDict): 460 """ 461 Generate bash script using template found next to this file. 462 This file is read in at constructor time to avoid file I/O 463 """ 464 # runscript_dir directory has to be consistent with gmakefile 465 testDict=srcDict[testname] 466 rpath=self.srcrelpath(root) 467 runscript_dir=os.path.join(self.testroot_dir,rpath) 468 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 469 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 470 471 # Get variables to go into shell scripts. last time testDict used 472 subst=self.getSubstVars(testDict,rpath,testname) 473 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 474 #if '33_' in testname: print subst['subargs'] 475 476 #Handle runfiles 477 for lfile in subst.get('localrunfiles','').split(): 478 fullfile=os.path.join(root,lfile) 479 if os.path.isdir(fullfile): 480 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 481 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 482 else: 483 shutil.copy(fullfile,runscript_dir) 484 # Check subtests for local runfiles 485 for stest in subst.get("subtests",[]): 486 for lfile in testDict[stest].get('localrunfiles','').split(): 487 fullfile=os.path.join(root,lfile) 488 if os.path.isdir(fullfile): 489 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 490 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 491 else: 492 shutil.copy(fullfile,self.runscript_dir) 493 494 # Now substitute the key variables into the header and footer 495 header=self._substVars(subst,example_template.header) 496 # The header is done twice to enable @...@ in header 497 header=self._substVars(subst,header) 498 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 499 500 # Start writing the file 501 fh.write(header+"\n") 502 503 # If there is a TODO or a SKIP then we do it before writing out the 504 # rest of the command (which is useful for working on the test) 505 # SKIP and TODO can be for the source file or for the runs 506 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 507 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 508 509 j=0 # for indentation 510 511 if loopVars: 512 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 513 if (loopHead): fh.write(loopHead+"\n") 514 515 # Subtests are special 516 if 'subtests' in testDict: 517 substP=subst # Subtests can inherit args but be careful 518 k=0 # for label suffixes 519 for stest in testDict["subtests"]: 520 subst=substP.copy() 521 subst.update(testDict[stest]) 522 # nsize is special because it is usually overwritten 523 if 'nsize' in testDict[stest]: 524 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 525 else: 526 fh.write("nsize=1\n") 527 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 528 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 529 #if '10_9' in testname: print sLoopVars 530 if sLoopVars: 531 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 532 fh.write(sLoopHead+"\n") 533 fh.write(self.getCmds(subst,j)+"\n") 534 if sLoopVars: 535 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 536 fh.write(sLoopFoot+"\n") 537 else: 538 fh.write(self.getCmds(subst,j)+"\n") 539 540 if loopVars: 541 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 542 fh.write(loopFoot+"\n") 543 544 fh.write(footer+"\n") 545 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 546 #if '10_9' in testname: sys.exit() 547 return 548 549 def genScriptsAndInfo(self,exfile,root,srcDict): 550 """ 551 Generate scripts from the source file, determine if built, etc. 552 For every test in the exfile with info in the srcDict: 553 1. Determine if it needs to be run for this arch 554 2. Generate the script 555 3. Generate the data needed to write out the makefile in a 556 convenient way 557 All tests are *always* run, but some may be SKIP'd per the TAP standard 558 """ 559 debug=False 560 execname=self.getExecname(exfile,root) 561 isBuilt=self._isBuilt(exfile,srcDict) 562 for test in srcDict: 563 if test in self.buildkeys: continue 564 if debug: print self.nameSpace(exfile,root), test 565 srcDict[test]['execname']=execname # Convenience in generating scripts 566 isRun=self._isRun(srcDict[test]) 567 self.genRunScript(test,root,isRun,srcDict) 568 srcDict[test]['isrun']=isRun 569 self.addToTests(test,root,exfile,execname,srcDict[test]) 570 571 # This adds to datastructure for building deps 572 if isBuilt: self.addToSources(exfile,root,srcDict) 573 return 574 575 def _isBuilt(self,exfile,srcDict): 576 """ 577 Determine if this file should be built. 578 """ 579 # Get the language based on file extension 580 srcDict['SKIP'] = [] 581 lang=self.getLanguage(exfile) 582 if (lang=="F" or lang=="F90"): 583 if not self.have_fortran: 584 srcDict["SKIP"].append("Fortran required for this test") 585 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 586 srcDict["SKIP"].append("Fortran f90freeform required for this test") 587 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 588 srcDict["SKIP"].append("CUDA required for this test") 589 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 590 srcDict["SKIP"].append("C++ required for this test") 591 592 # Deprecated source files 593 if srcDict.get("TODO"): 594 return False 595 596 # isRun can work with srcDict to handle the requires 597 if "requires" in srcDict: 598 if srcDict["requires"]: 599 return self._isRun(srcDict) 600 601 return srcDict['SKIP'] == [] 602 603 604 def _isRun(self,testDict, debug=False): 605 """ 606 Based on the requirements listed in the src file and the petscconf.h 607 info, determine whether this test should be run or not. 608 """ 609 indent=" " 610 611 if 'SKIP' not in testDict: 612 testDict['SKIP'] = [] 613 # MPI requirements 614 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 615 if debug: print indent+"Cannot run parallel tests" 616 testDict['SKIP'].append("Parallel test with serial build") 617 618 # The requirements for the test are the sum of all the run subtests 619 if 'subtests' in testDict: 620 if 'requires' not in testDict: testDict['requires']="" 621 for stest in testDict['subtests']: 622 if 'requires' in testDict[stest]: 623 testDict['requires']+=" "+testDict[stest]['requires'] 624 625 626 # Now go through all requirements 627 if 'requires' in testDict: 628 for requirement in testDict['requires'].split(): 629 requirement=requirement.strip() 630 if not requirement: continue 631 if debug: print indent+"Requirement: ", requirement 632 isNull=False 633 if requirement.startswith("!"): 634 requirement=requirement[1:]; isNull=True 635 # Precision requirement for reals 636 if requirement in self.precision_types: 637 if self.conf['PETSC_PRECISION']==requirement: 638 if isNull: 639 testDict['SKIP'].append("not "+requirement+" required") 640 continue 641 continue # Success 642 elif not isNull: 643 testDict['SKIP'].append(requirement+" required") 644 continue 645 # Precision requirement for ints 646 if requirement in self.integer_types: 647 if requirement=="int32": 648 if self.conf['PETSC_SIZEOF_INT']==4: 649 if isNull: 650 testDict['SKIP'].append("not int32 required") 651 continue 652 continue # Success 653 elif not isNull: 654 testDict['SKIP'].append("int32 required") 655 continue 656 if requirement=="int64": 657 if self.conf['PETSC_SIZEOF_INT']==8: 658 if isNull: 659 testDict['SKIP'].append("NOT int64 required") 660 continue 661 continue # Success 662 elif not isNull: 663 testDict['SKIP'].append("int64 required") 664 continue 665 # Datafilespath 666 if requirement=="datafilespath" and not isNull: 667 testDict['SKIP'].append("Requires DATAFILESPATH") 668 continue 669 # Defines -- not sure I have comments matching 670 if "define(" in requirement.lower(): 671 reqdef=requirement.split("(")[1].split(")")[0] 672 if reqdef in self.conf: 673 if isNull: 674 testDict['SKIP'].append("Null requirement not met: "+requirement) 675 continue 676 continue # Success 677 elif not isNull: 678 testDict['SKIP'].append("Required: "+requirement) 679 continue 680 681 # Rest should be packages that we can just get from conf 682 if requirement == "complex": 683 petscconfvar="PETSC_USE_COMPLEX" 684 else: 685 petscconfvar="PETSC_HAVE_"+requirement.upper() 686 if self.conf.get(petscconfvar): 687 if isNull: 688 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 689 continue 690 continue # Success 691 elif not isNull: 692 if debug: print "requirement not found: ", requirement 693 testDict['SKIP'].append(petscconfvar+" requirement not met") 694 continue 695 696 return testDict['SKIP'] == [] 697 698 def genPetscTests_summarize(self,dataDict): 699 """ 700 Required method to state what happened 701 """ 702 if not self.summarize: return 703 indent=" " 704 print(self.testroot_dir, os.path.realpath(os.curdir)) 705 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 706 fh=open(fhname,"w") 707 #print "See ", fhname 708 for root in dataDict: 709 relroot=self.srcrelpath(root) 710 pkg=relroot.split("/")[1] 711 fh.write(relroot+"\n") 712 allSrcs=[] 713 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 714 for exfile in dataDict[root]: 715 # Basic information 716 rfile=os.path.join(relroot,exfile) 717 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 718 fh.write(indent+exfile+indent*4+builtStatus+"\n") 719 720 for test in dataDict[root][exfile]: 721 if test in self.buildkeys: continue 722 line=indent*2+test 723 fh.write(line+"\n") 724 # Looks nice to have the keys in order 725 #for key in dataDict[root][exfile][test]: 726 for key in "isrun abstracted nsize args requires script".split(): 727 if key not in dataDict[root][exfile][test]: continue 728 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 729 fh.write(line+"\n") 730 fh.write("\n") 731 fh.write("\n") 732 fh.write("\n") 733 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 734 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 735 fh.close() 736 return 737 738 def genPetscTests(self,root,dirs,files,dataDict): 739 """ 740 Go through and parse the source files in the directory to generate 741 the examples based on the metadata contained in the source files 742 """ 743 debug=False 744 # Use examplesAnalyze to get what the makefles think are sources 745 #self.examplesAnalyze(root,dirs,files,anlzDict) 746 747 dataDict[root]={} 748 749 for exfile in files: 750 #TST: Until we replace files, still leaving the orginals as is 751 #if not exfile.startswith("new_"+"ex"): continue 752 #if not exfile.startswith("ex"): continue 753 754 # Ignore emacs and other temporary files 755 if exfile.startswith("."): continue 756 if exfile.startswith("#"): continue 757 758 # Convenience 759 fullex=os.path.join(root,exfile) 760 if self.verbose: print(' --> '+fullex) 761 dataDict[root].update(testparse.parseTestFile(fullex,0)) 762 if exfile in dataDict[root]: 763 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 764 765 return 766 767 def walktree(self,top): 768 """ 769 Walk a directory tree, starting from 'top' 770 """ 771 #print "action", action 772 # Goal of action is to fill this dictionary 773 dataDict={} 774 for root, dirs, files in os.walk(top, topdown=False): 775 if not "examples" in root: continue 776 if not os.path.isfile(os.path.join(root,"makefile")): continue 777 if self.verbose: print(root) 778 bname=os.path.basename(root.rstrip("/")) 779 self.genPetscTests(root,dirs,files,dataDict) 780 # Now summarize this dictionary 781 self.genPetscTests_summarize(dataDict) 782 return dataDict 783 784 def gen_gnumake(self, fd): 785 """ 786 Overwrite of the method in the base PETSc class 787 """ 788 def write(stem, srcs): 789 for lang in LANGS: 790 if srcs[lang]['srcs']: 791 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 792 for pkg in PKGS: 793 srcs = self.gen_pkg(pkg) 794 write('testsrcs-' + pkg, srcs) 795 # Handle dependencies 796 for lang in LANGS: 797 for exfile in srcs[lang]['srcs']: 798 if exfile in srcs[lang]: 799 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 800 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 801 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 802 if deps: 803 # The executable literally depends on the object file because it is linked 804 fd.write(ex +": " + " ".join(deps) +'\n') 805 # The object file containing 'main' does not normally depend on other object 806 # files, but it does when it includes their modules. This dependency is 807 # overly blunt and could be reduced to only depend on object files for 808 # modules that are used, like "*f90aux.o". 809 fd.write(exfo +": " + " ".join(deps) +'\n') 810 811 return self.gendeps 812 813 def gen_pkg(self, pkg): 814 """ 815 Overwrite of the method in the base PETSc class 816 """ 817 return self.sources[pkg] 818 819 def write_gnumake(self,dataDict): 820 """ 821 Write out something similar to files from gmakegen.py 822 823 Test depends on script which also depends on source 824 file, but since I don't have a good way generating 825 acting on a single file (oops) just depend on 826 executable which in turn will depend on src file 827 """ 828 # Different options for how to set up the targets 829 compileExecsFirst=False 830 831 # Open file 832 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 833 fd = open(arch_files, 'w') 834 835 # Write out the sources 836 gendeps = self.gen_gnumake(fd) 837 838 # Write out the tests and execname targets 839 fd.write("\n#Tests and executables\n") # Delimiter 840 841 for pkg in PKGS: 842 # These grab the ones that are built 843 for lang in LANGS: 844 testdeps=[] 845 for ftest in self.tests[pkg][lang]: 846 test=os.path.basename(ftest) 847 basedir=os.path.dirname(ftest) 848 testdeps.append(self.nameSpace(test,basedir)) 849 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 850 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 851 852 # test targets 853 for ftest in self.tests[pkg][lang]: 854 test=os.path.basename(ftest) 855 basedir=os.path.dirname(ftest) 856 testdir="${TESTDIR}/"+basedir+"/" 857 nmtest=self.nameSpace(test,basedir) 858 rundir=os.path.join(testdir,test) 859 #print test, nmtest 860 script=test+".sh" 861 862 # Deps 863 exfile=self.tests[pkg][lang][ftest]['exfile'] 864 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 865 localexec=self.tests[pkg][lang][ftest]['exec'] 866 execname=os.path.join(testdir,localexec) 867 fullscript=os.path.join(testdir,script) 868 tmpfile=os.path.join(testdir,test,test+".tmp") 869 870 # *.counts depends on the script and either executable (will 871 # be run) or the example source file (SKIP or TODO) 872 fd.write('%s.counts : %s %s' 873 % (os.path.join('$(TESTDIR)/counts', nmtest), 874 fullscript, 875 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 876 ) 877 if exfile in self.sources[pkg][lang]: 878 for dep in self.sources[pkg][lang][exfile]: 879 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 880 fd.write('\n') 881 882 # Now write the args: 883 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 884 885 fd.close() 886 return 887 888 def writeHarness(self,output,dataDict): 889 """ 890 This is set up to write out multiple harness even if only gnumake 891 is supported now 892 """ 893 eval("self.write_"+output+"(dataDict)") 894 return 895 896def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 897 if output is None: 898 output = 'gnumake' 899 900 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 901 if petsc_arch: 902 if len(petsc_arch.split(os.path.sep))>1: 903 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 904 905 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 906 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 907 testdir=testdir) 908 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 909 pEx.writeHarness(output,dataDict) 910 911if __name__ == '__main__': 912 import optparse 913 parser = optparse.OptionParser() 914 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 915 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 916 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 917 parser.add_option('--output', help='Location to write output file', default=None) 918 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 919 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 920 opts, extra_args = parser.parse_args() 921 opts, extra_args = parser.parse_args() 922 if extra_args: 923 import sys 924 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 925 exit(1) 926 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 927 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 928