1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19class generateExamples(Petsc): 20 """ 21 gmakegen.py has basic structure for finding the files, writing out 22 the dependencies, etc. 23 """ 24 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 25 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 26 27 self.single_ex=single_ex 28 29 # Set locations to handle movement 30 self.inInstallDir=self.getInInstallDir(thisscriptdir) 31 if not self.inInstallDir: 32 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 33 self.srcdir=os.path.join(self.petsc_dir,'src') 34 else: 35 # set PETSC_ARCH to install directory to allow script to work in both 36 dirlist=thisscriptdir.split(os.path.sep) 37 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 38 self.arch_dir=installdir 39 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 40 41 # Do some initialization 42 if testdir: 43 # If full path given, then use it, otherwise assume relative to arch_dir 44 if testdir.strip().startswith(os.path.sep): 45 self.testroot_dir=testdir.strip() 46 else: 47 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 48 else: 49 self.testroot_dir=os.path.join(self.arch_dir,"tests") 50 51 self.ptNaming=True 52 self.verbose=verbose 53 # Whether to write out a useful debugging 54 self.summarize=True if verbose else False 55 56 # For help in setting the requirements 57 self.precision_types="single double __float128 int32".split() 58 self.integer_types="int32 int64".split() 59 self.languages="fortran cuda cxx".split() # Always requires C so do not list 60 61 # Things that are not test 62 self.buildkeys=testparse.buildkeys 63 64 # Adding a dictionary for storing sources, objects, and tests 65 # to make building the dependency tree easier 66 self.sources={} 67 self.objects={} 68 self.tests={} 69 for pkg in PKGS: 70 self.sources[pkg]={} 71 self.objects[pkg]=[] 72 self.tests[pkg]={} 73 for lang in LANGS: 74 self.sources[pkg][lang]={} 75 self.sources[pkg][lang]['srcs']=[] 76 self.tests[pkg][lang]={} 77 78 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 79 80 self.indent=" " 81 if self.verbose: print('Finishing the constructor') 82 return 83 84 def srcrelpath(self,rdir): 85 """ 86 Get relative path to source directory 87 """ 88 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 89 90 def getInInstallDir(self,thisscriptdir): 91 """ 92 When petsc is installed then this file in installed in: 93 <PREFIX>/share/petsc/examples/config/gmakegentest.py 94 otherwise the path is: 95 <PETSC_DIR>/config/gmakegentest.py 96 We use this difference to determine if we are in installdir 97 """ 98 dirlist=thisscriptdir.split(os.path.sep) 99 if len(dirlist)>4: 100 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 101 if lastfour==os.path.join('share','petsc','examples','config'): 102 return True 103 else: 104 return False 105 else: 106 return False 107 108 def nameSpace(self,srcfile,srcdir): 109 """ 110 Because the scripts have a non-unique naming, the pretty-printing 111 needs to convey the srcdir and srcfile. There are two ways of doing this. 112 """ 113 if self.ptNaming: 114 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 115 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 116 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 117 nameString=prefix+srcfile 118 else: 119 #nameString=srcdir+": "+srcfile 120 nameString=srcfile 121 return nameString 122 123 def getLanguage(self,srcfile): 124 """ 125 Based on the source, determine associated language as found in gmakegen.LANGS 126 Can we just return srcext[1:\] now? 127 """ 128 langReq=None 129 srcext=os.path.splitext(srcfile)[-1] 130 if srcext in ".F90".split(): langReq="F90" 131 if srcext in ".F".split(): langReq="F" 132 if srcext in ".cxx".split(): langReq="cxx" 133 if srcext == ".cu": langReq="cu" 134 if srcext == ".c": langReq="c" 135 #if not langReq: print "ERROR: ", srcext, srcfile 136 return langReq 137 138 def _getLoopVars(self,inDict,testname, isSubtest=False): 139 """ 140 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 141 Return: 142 inDict['args']: -ksp_monitor 143 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 144 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 145 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 146 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 147 subst should be passed in instead of inDict 148 """ 149 loopVars={}; newargs="" 150 lkeys=inDict.keys() 151 lsuffix='_' 152 from testparse import parseLoopArgs 153 for key in lkeys: 154 if type(inDict[key])!=types.StringType: continue 155 keystr = str(inDict[key]) 156 akey=('subargs' if key=='args' else key) # what to assign 157 if akey not in inDict: inDict[akey]='' 158 varlist=[] 159 for varset in re.split('-(?=[a-zA-Z])',keystr): 160 if not varset.strip(): continue 161 if '{{' in varset: 162 keyvar,lvars,ftype=parseLoopArgs(varset) 163 if akey not in loopVars: loopVars[akey]={} 164 varlist.append(keyvar) 165 loopVars[akey][keyvar]=[keyvar,lvars] 166 if akey=='nsize': 167 inDict[akey] = '${' + keyvar + '}' 168 lsuffix+=akey+'-'+inDict[akey]+'_' 169 else: 170 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 171 lsuffix+=keyvar+'-${' + keyvar + '}_' 172 else: 173 if key=='args': newargs+=" -"+varset.strip() 174 if varlist: loopVars[akey]['varlist']=varlist 175 176 177 # For subtests, args are always substituted in (not top level) 178 if isSubtest: 179 inDict['subargs']+=" "+newargs.strip() 180 inDict['args']='' 181 if 'label_suffix' in inDict: 182 inDict['label_suffix']+=lsuffix.rstrip('_') 183 else: 184 inDict['label_suffix']=lsuffix.rstrip('_') 185 else: 186 if loopVars.keys(): 187 inDict['args']=newargs.strip() 188 inDict['label_suffix']=lsuffix.rstrip('_') 189 if loopVars.keys(): 190 return loopVars 191 else: 192 return None 193 194 def getArgLabel(self,testDict): 195 """ 196 In all of the arguments in the test dictionary, create a simple 197 string for searching within the makefile system. For simplicity in 198 search, remove "-", for strings, etc. 199 Also, concatenate the arg commands 200 For now, ignore nsize -- seems hard to search for anyway 201 """ 202 # Collect all of the args associated with a test 203 argStr=("" if 'args' not in testDict else testDict['args']) 204 if 'subtests' in testDict: 205 for stest in testDict["subtests"]: 206 sd=testDict[stest] 207 argStr=argStr+("" if 'args' not in sd else sd['args']) 208 209 # Now go through and cleanup 210 argStr=re.sub('{{(.*?)}}',"",argStr) 211 argStr=re.sub('-'," ",argStr) 212 for digit in string.digits: argStr=re.sub(digit," ",argStr) 213 argStr=re.sub("\.","",argStr) 214 argStr=re.sub(",","",argStr) 215 argStr=re.sub('\+',' ',argStr) 216 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 217 return argStr.strip() 218 219 def addToSources(self,exfile,root,srcDict): 220 """ 221 Put into data structure that allows easy generation of makefile 222 """ 223 rpath=self.srcrelpath(root) 224 pkg=rpath.split(os.path.sep)[1] 225 relpfile=os.path.join(rpath,exfile) 226 lang=self.getLanguage(exfile) 227 if not lang: return 228 self.sources[pkg][lang]['srcs'].append(relpfile) 229 self.sources[pkg][lang][relpfile] = [] 230 if 'depends' in srcDict: 231 depSrcList=srcDict['depends'].split() 232 for depSrc in depSrcList: 233 depObj=os.path.splitext(depSrc)[0]+".o" 234 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 235 236 # In gmakefile, ${TESTDIR} var specifies the object compilation 237 testsdir=self.srcrelpath(root)+"/" 238 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 239 self.objects[pkg].append(objfile) 240 return 241 242 def addToTests(self,test,root,exfile,execname,testDict): 243 """ 244 Put into data structure that allows easy generation of makefile 245 Organized by languages to allow testing of languages 246 """ 247 rpath=self.srcrelpath(root) 248 pkg=rpath.split("/")[1] 249 #nmtest=self.nameSpace(test,root) 250 nmtest=os.path.join(rpath,test) 251 lang=self.getLanguage(exfile) 252 if not lang: return 253 self.tests[pkg][lang][nmtest]={} 254 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 255 self.tests[pkg][lang][nmtest]['exec']=execname 256 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 257 return 258 259 def getExecname(self,exfile,root): 260 """ 261 Generate bash script using template found next to this file. 262 This file is read in at constructor time to avoid file I/O 263 """ 264 rpath=self.srcrelpath(root) 265 if self.single_ex: 266 execname=rpath.split("/")[1]+"-ex" 267 else: 268 execname=os.path.splitext(exfile)[0] 269 return execname 270 271 def getSubstVars(self,testDict,rpath,testname): 272 """ 273 Create a dictionary with all of the variables that get substituted 274 into the template commands found in example_template.py 275 """ 276 subst={} 277 278 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 279 if 'nsize' not in testDict: testDict['nsize']=1 280 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 281 for ak in testparse.acceptedkeys: 282 if ak=='test': continue 283 subst[ak]=(testDict[ak] if ak in testDict else '') 284 285 # Now do other variables 286 subst['execname']=testDict['execname'] 287 if 'filter' in testDict: 288 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 289 290 # Others 291 subst['subargs']='' # Default. For variables override 292 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 293 subst['label_suffix']='' 294 subst['comments']="\n#".join(subst['comments'].split("\n")) 295 if subst['comments']: subst['comments']="#"+subst['comments'] 296 subst['exec']="../"+subst['execname'] 297 subst['testroot']=self.testroot_dir 298 subst['testname']=testname 299 dp = self.conf.get('DATAFILESPATH','') 300 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 301 302 # This is used to label some matrices 303 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 304 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 305 306 # These can have for loops and are treated separately later 307 subst['nsize']=str(subst['nsize']) 308 309 #Conf vars 310 if self.petsc_arch.find('valgrind')>=0: 311 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 312 else: 313 subst['mpiexec']=self.conf['MPIEXEC'] 314 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 315 subst['petsc_arch']=self.petsc_arch 316 if not self.inInstallDir: 317 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 318 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 319 else: 320 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 321 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 322 subst['diff']=self.conf['DIFF'] 323 subst['rm']=self.conf['RM'] 324 subst['grep']=self.conf['GREP'] 325 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 326 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 327 328 # Output file is special because of subtests override 329 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 330 if not "_" in defroot: defroot=defroot+"_1" 331 subst['defroot']=defroot 332 subst['label']=self.nameSpace(defroot,subst['srcdir']) 333 subst['redirect_file']=defroot+".tmp" 334 if 'output_file' not in testDict: 335 subst['output_file']="output/"+defroot+".out" 336 # Add in the full path here. 337 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 338 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 339 if not subst['TODO']: 340 print "Warning: "+subst['output_file']+" not found." 341 # Worry about alt files here -- see 342 # src/snes/examples/tutorials/output/ex22*.out 343 altlist=[subst['output_file']] 344 basefile,ext = os.path.splitext(subst['output_file']) 345 for i in range(1,9): 346 altroot=basefile+"_alt" 347 if i > 1: altroot=altroot+"_"+str(i) 348 af=altroot+".out" 349 srcaf=os.path.join(subst['srcdir'],af) 350 fullaf=os.path.join(self.petsc_dir,srcaf) 351 if os.path.isfile(fullaf): altlist.append(srcaf) 352 if len(altlist)>1: subst['altfiles']=altlist 353 #if len(altlist)>1: print "Found alt files: ",altlist 354 355 return subst 356 357 def getCmds(self,subst,i): 358 """ 359 Generate bash script using template found next to this file. 360 This file is read in at constructor time to avoid file I/O 361 """ 362 nindnt=i # the start and has to be consistent with below 363 cmdindnt=self.indent*nindnt 364 cmdLines="" 365 366 # MPI is the default -- but we have a few odd commands 367 if not subst['command']: 368 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 369 else: 370 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 371 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 372 373 cmdLines+=cmdindnt+'if test $res = 0; then\n' 374 diffindnt=self.indent*(nindnt+1) 375 if not subst['filter_output']: 376 if 'altfiles' not in subst: 377 cmd=diffindnt+self._substVars(subst,example_template.difftest) 378 else: 379 # Have to do it by hand a bit because of variable number of alt files 380 rf=subst['redirect_file'] 381 cmd=diffindnt+example_template.difftest.split('@')[0] 382 for i in range(len(subst['altfiles'])): 383 af=subst['altfiles'][i] 384 cmd+=af+' '+rf 385 if i!=len(subst['altfiles'])-1: 386 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 387 cmd+=' || ${diff_exe} ' 388 else: 389 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 390 cmd+=subst['label_suffix']+' ""' # Quotes are painful 391 else: 392 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 393 cmdLines+=cmd+"\n" 394 cmdLines+=cmdindnt+'else\n' 395 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 396 cmdLines+=cmdindnt+'fi\n' 397 return cmdLines 398 399 def _substVars(self,subst,origStr): 400 """ 401 Substitute variables 402 """ 403 Str=origStr 404 for subkey in subst: 405 if type(subst[subkey])!=types.StringType: continue 406 patt="@"+subkey.upper()+"@" 407 Str=re.sub(patt,subst[subkey],Str) 408 return Str 409 410 def _writeTodoSkip(self,fh,tors,reasons,footer): 411 """ 412 Write out the TODO and SKIP lines in the file 413 The TODO or SKIP variable, tors, should be lower case 414 """ 415 TORS=tors.upper() 416 template=eval("example_template."+tors+"line") 417 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 418 tab = '' 419 if reasons: 420 fh.write('if ! $force; then\n') 421 tab = tab + ' ' 422 if reasons == ["Requires DATAFILESPATH"]: 423 # The only reason not to run is DATAFILESPATH, which we check at run-time 424 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 425 tab = tab + ' ' 426 if reasons: 427 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 428 fh.write(tab+footer+"\n") 429 fh.write(tab+"exit\n") 430 if reasons == ["Requires DATAFILESPATH"]: 431 fh.write(' fi\n') 432 if reasons: 433 fh.write('fi\n') 434 fh.write('\n\n') 435 return 436 437 def getLoopVarsHead(self,loopVars,i): 438 """ 439 Generate a nicely indented string with the format loops 440 Here is what the data structure looks like 441 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 442 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 443 loopVars['subargs']['pc_type']=["j","cholesky sor"] 444 """ 445 outstr=''; indnt=self.indent 446 for key in loopVars: 447 for var in loopVars[key]['varlist']: 448 varval=loopVars[key][var] 449 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 450 i = i + 1 451 return (outstr,i) 452 453 def getLoopVarsFoot(self,loopVars,i): 454 outstr=''; indnt=self.indent 455 for key in loopVars: 456 for var in loopVars[key]['varlist']: 457 i = i - 1 458 outstr += indnt * i + "done\n" 459 return (outstr,i) 460 461 def genRunScript(self,testname,root,isRun,srcDict): 462 """ 463 Generate bash script using template found next to this file. 464 This file is read in at constructor time to avoid file I/O 465 """ 466 # runscript_dir directory has to be consistent with gmakefile 467 testDict=srcDict[testname] 468 rpath=self.srcrelpath(root) 469 runscript_dir=os.path.join(self.testroot_dir,rpath) 470 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 471 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 472 473 # Get variables to go into shell scripts. last time testDict used 474 subst=self.getSubstVars(testDict,rpath,testname) 475 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 476 #if '33_' in testname: print subst['subargs'] 477 478 #Handle runfiles 479 for lfile in subst.get('localrunfiles','').split(): 480 fullfile=os.path.join(root,lfile) 481 if os.path.isdir(fullfile): 482 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 483 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 484 else: 485 shutil.copy(fullfile,runscript_dir) 486 # Check subtests for local runfiles 487 for stest in subst.get("subtests",[]): 488 for lfile in testDict[stest].get('localrunfiles','').split(): 489 fullfile=os.path.join(root,lfile) 490 if os.path.isdir(fullfile): 491 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 492 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 493 else: 494 shutil.copy(fullfile,self.runscript_dir) 495 496 # Now substitute the key variables into the header and footer 497 header=self._substVars(subst,example_template.header) 498 # The header is done twice to enable @...@ in header 499 header=self._substVars(subst,header) 500 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 501 502 # Start writing the file 503 fh.write(header+"\n") 504 505 # If there is a TODO or a SKIP then we do it before writing out the 506 # rest of the command (which is useful for working on the test) 507 # SKIP and TODO can be for the source file or for the runs 508 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 509 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 510 511 j=0 # for indentation 512 513 if loopVars: 514 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 515 if (loopHead): fh.write(loopHead+"\n") 516 517 # Subtests are special 518 if 'subtests' in testDict: 519 substP=subst # Subtests can inherit args but be careful 520 k=0 # for label suffixes 521 for stest in testDict["subtests"]: 522 subst=substP.copy() 523 subst.update(testDict[stest]) 524 # nsize is special because it is usually overwritten 525 if 'nsize' in testDict[stest]: 526 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 527 else: 528 fh.write("nsize=1\n") 529 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 530 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 531 #if '10_9' in testname: print sLoopVars 532 if sLoopVars: 533 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 534 fh.write(sLoopHead+"\n") 535 fh.write(self.getCmds(subst,j)+"\n") 536 if sLoopVars: 537 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 538 fh.write(sLoopFoot+"\n") 539 else: 540 fh.write(self.getCmds(subst,j)+"\n") 541 542 if loopVars: 543 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 544 fh.write(loopFoot+"\n") 545 546 fh.write(footer+"\n") 547 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 548 #if '10_9' in testname: sys.exit() 549 return 550 551 def genScriptsAndInfo(self,exfile,root,srcDict): 552 """ 553 Generate scripts from the source file, determine if built, etc. 554 For every test in the exfile with info in the srcDict: 555 1. Determine if it needs to be run for this arch 556 2. Generate the script 557 3. Generate the data needed to write out the makefile in a 558 convenient way 559 All tests are *always* run, but some may be SKIP'd per the TAP standard 560 """ 561 debug=False 562 execname=self.getExecname(exfile,root) 563 isBuilt=self._isBuilt(exfile,srcDict) 564 for test in srcDict: 565 if test in self.buildkeys: continue 566 if debug: print self.nameSpace(exfile,root), test 567 srcDict[test]['execname']=execname # Convenience in generating scripts 568 isRun=self._isRun(srcDict[test]) 569 self.genRunScript(test,root,isRun,srcDict) 570 srcDict[test]['isrun']=isRun 571 self.addToTests(test,root,exfile,execname,srcDict[test]) 572 573 # This adds to datastructure for building deps 574 if isBuilt: self.addToSources(exfile,root,srcDict) 575 return 576 577 def _isBuilt(self,exfile,srcDict): 578 """ 579 Determine if this file should be built. 580 """ 581 # Get the language based on file extension 582 srcDict['SKIP'] = [] 583 lang=self.getLanguage(exfile) 584 if (lang=="F" or lang=="F90"): 585 if not self.have_fortran: 586 srcDict["SKIP"].append("Fortran required for this test") 587 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 588 srcDict["SKIP"].append("Fortran f90freeform required for this test") 589 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 590 srcDict["SKIP"].append("CUDA required for this test") 591 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 592 srcDict["SKIP"].append("C++ required for this test") 593 594 # Deprecated source files 595 if srcDict.get("TODO"): 596 return False 597 598 # isRun can work with srcDict to handle the requires 599 if "requires" in srcDict: 600 if srcDict["requires"]: 601 return self._isRun(srcDict) 602 603 return srcDict['SKIP'] == [] 604 605 606 def _isRun(self,testDict, debug=False): 607 """ 608 Based on the requirements listed in the src file and the petscconf.h 609 info, determine whether this test should be run or not. 610 """ 611 indent=" " 612 613 if 'SKIP' not in testDict: 614 testDict['SKIP'] = [] 615 # MPI requirements 616 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 617 if debug: print indent+"Cannot run parallel tests" 618 testDict['SKIP'].append("Parallel test with serial build") 619 620 # The requirements for the test are the sum of all the run subtests 621 if 'subtests' in testDict: 622 if 'requires' not in testDict: testDict['requires']="" 623 for stest in testDict['subtests']: 624 if 'requires' in testDict[stest]: 625 testDict['requires']+=" "+testDict[stest]['requires'] 626 627 628 # Now go through all requirements 629 if 'requires' in testDict: 630 for requirement in testDict['requires'].split(): 631 requirement=requirement.strip() 632 if not requirement: continue 633 if debug: print indent+"Requirement: ", requirement 634 isNull=False 635 if requirement.startswith("!"): 636 requirement=requirement[1:]; isNull=True 637 # Precision requirement for reals 638 if requirement in self.precision_types: 639 if self.conf['PETSC_PRECISION']==requirement: 640 if isNull: 641 testDict['SKIP'].append("not "+requirement+" required") 642 continue 643 continue # Success 644 elif not isNull: 645 testDict['SKIP'].append(requirement+" required") 646 continue 647 # Precision requirement for ints 648 if requirement in self.integer_types: 649 if requirement=="int32": 650 if self.conf['PETSC_SIZEOF_INT']==4: 651 if isNull: 652 testDict['SKIP'].append("not int32 required") 653 continue 654 continue # Success 655 elif not isNull: 656 testDict['SKIP'].append("int32 required") 657 continue 658 if requirement=="int64": 659 if self.conf['PETSC_SIZEOF_INT']==8: 660 if isNull: 661 testDict['SKIP'].append("NOT int64 required") 662 continue 663 continue # Success 664 elif not isNull: 665 testDict['SKIP'].append("int64 required") 666 continue 667 # Datafilespath 668 if requirement=="datafilespath" and not isNull: 669 testDict['SKIP'].append("Requires DATAFILESPATH") 670 continue 671 # Defines -- not sure I have comments matching 672 if "define(" in requirement.lower(): 673 reqdef=requirement.split("(")[1].split(")")[0] 674 if reqdef in self.conf: 675 if isNull: 676 testDict['SKIP'].append("Null requirement not met: "+requirement) 677 continue 678 continue # Success 679 elif not isNull: 680 testDict['SKIP'].append("Required: "+requirement) 681 continue 682 683 # Rest should be packages that we can just get from conf 684 if requirement == "complex": 685 petscconfvar="PETSC_USE_COMPLEX" 686 else: 687 petscconfvar="PETSC_HAVE_"+requirement.upper() 688 if self.conf.get(petscconfvar): 689 if isNull: 690 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 691 continue 692 continue # Success 693 elif not isNull: 694 if debug: print "requirement not found: ", requirement 695 testDict['SKIP'].append(petscconfvar+" requirement not met") 696 continue 697 698 return testDict['SKIP'] == [] 699 700 def genPetscTests_summarize(self,dataDict): 701 """ 702 Required method to state what happened 703 """ 704 if not self.summarize: return 705 indent=" " 706 print(self.testroot_dir, os.path.realpath(os.curdir)) 707 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 708 fh=open(fhname,"w") 709 #print "See ", fhname 710 for root in dataDict: 711 relroot=self.srcrelpath(root) 712 pkg=relroot.split("/")[1] 713 fh.write(relroot+"\n") 714 allSrcs=[] 715 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 716 for exfile in dataDict[root]: 717 # Basic information 718 rfile=os.path.join(relroot,exfile) 719 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 720 fh.write(indent+exfile+indent*4+builtStatus+"\n") 721 722 for test in dataDict[root][exfile]: 723 if test in self.buildkeys: continue 724 line=indent*2+test 725 fh.write(line+"\n") 726 # Looks nice to have the keys in order 727 #for key in dataDict[root][exfile][test]: 728 for key in "isrun abstracted nsize args requires script".split(): 729 if key not in dataDict[root][exfile][test]: continue 730 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 731 fh.write(line+"\n") 732 fh.write("\n") 733 fh.write("\n") 734 fh.write("\n") 735 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 736 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 737 fh.close() 738 return 739 740 def genPetscTests(self,root,dirs,files,dataDict): 741 """ 742 Go through and parse the source files in the directory to generate 743 the examples based on the metadata contained in the source files 744 """ 745 debug=False 746 # Use examplesAnalyze to get what the makefles think are sources 747 #self.examplesAnalyze(root,dirs,files,anlzDict) 748 749 dataDict[root]={} 750 751 for exfile in files: 752 #TST: Until we replace files, still leaving the orginals as is 753 #if not exfile.startswith("new_"+"ex"): continue 754 #if not exfile.startswith("ex"): continue 755 756 # Ignore emacs and other temporary files 757 if exfile.startswith("."): continue 758 if exfile.startswith("#"): continue 759 760 # Convenience 761 fullex=os.path.join(root,exfile) 762 if self.verbose: print(' --> '+fullex) 763 dataDict[root].update(testparse.parseTestFile(fullex,0)) 764 if exfile in dataDict[root]: 765 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 766 767 return 768 769 def walktree(self,top): 770 """ 771 Walk a directory tree, starting from 'top' 772 """ 773 #print "action", action 774 # Goal of action is to fill this dictionary 775 dataDict={} 776 for root, dirs, files in os.walk(top, topdown=True): 777 if not "examples" in root: continue 778 if "dSYM" in root: continue 779 if os.path.basename(root.rstrip("/")) == 'output': continue 780 if self.verbose: print(root) 781 self.genPetscTests(root,dirs,files,dataDict) 782 # Now summarize this dictionary 783 self.genPetscTests_summarize(dataDict) 784 return dataDict 785 786 def gen_gnumake(self, fd): 787 """ 788 Overwrite of the method in the base PETSc class 789 """ 790 def write(stem, srcs): 791 for lang in LANGS: 792 if srcs[lang]['srcs']: 793 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 794 for pkg in PKGS: 795 srcs = self.gen_pkg(pkg) 796 write('testsrcs-' + pkg, srcs) 797 # Handle dependencies 798 for lang in LANGS: 799 for exfile in srcs[lang]['srcs']: 800 if exfile in srcs[lang]: 801 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 802 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 803 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 804 if deps: 805 # The executable literally depends on the object file because it is linked 806 fd.write(ex +": " + " ".join(deps) +'\n') 807 # The object file containing 'main' does not normally depend on other object 808 # files, but it does when it includes their modules. This dependency is 809 # overly blunt and could be reduced to only depend on object files for 810 # modules that are used, like "*f90aux.o". 811 fd.write(exfo +": " + " ".join(deps) +'\n') 812 813 return self.gendeps 814 815 def gen_pkg(self, pkg): 816 """ 817 Overwrite of the method in the base PETSc class 818 """ 819 return self.sources[pkg] 820 821 def write_gnumake(self,dataDict): 822 """ 823 Write out something similar to files from gmakegen.py 824 825 Test depends on script which also depends on source 826 file, but since I don't have a good way generating 827 acting on a single file (oops) just depend on 828 executable which in turn will depend on src file 829 """ 830 # Different options for how to set up the targets 831 compileExecsFirst=False 832 833 # Open file 834 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 835 fd = open(arch_files, 'w') 836 837 # Write out the sources 838 gendeps = self.gen_gnumake(fd) 839 840 # Write out the tests and execname targets 841 fd.write("\n#Tests and executables\n") # Delimiter 842 843 for pkg in PKGS: 844 # These grab the ones that are built 845 for lang in LANGS: 846 testdeps=[] 847 for ftest in self.tests[pkg][lang]: 848 test=os.path.basename(ftest) 849 basedir=os.path.dirname(ftest) 850 testdeps.append(self.nameSpace(test,basedir)) 851 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 852 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 853 854 # test targets 855 for ftest in self.tests[pkg][lang]: 856 test=os.path.basename(ftest) 857 basedir=os.path.dirname(ftest) 858 testdir="${TESTDIR}/"+basedir+"/" 859 nmtest=self.nameSpace(test,basedir) 860 rundir=os.path.join(testdir,test) 861 #print test, nmtest 862 script=test+".sh" 863 864 # Deps 865 exfile=self.tests[pkg][lang][ftest]['exfile'] 866 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 867 localexec=self.tests[pkg][lang][ftest]['exec'] 868 execname=os.path.join(testdir,localexec) 869 fullscript=os.path.join(testdir,script) 870 tmpfile=os.path.join(testdir,test,test+".tmp") 871 872 # *.counts depends on the script and either executable (will 873 # be run) or the example source file (SKIP or TODO) 874 fd.write('%s.counts : %s %s' 875 % (os.path.join('$(TESTDIR)/counts', nmtest), 876 fullscript, 877 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 878 ) 879 if exfile in self.sources[pkg][lang]: 880 for dep in self.sources[pkg][lang][exfile]: 881 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 882 fd.write('\n') 883 884 # Now write the args: 885 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 886 887 fd.close() 888 return 889 890 def writeHarness(self,output,dataDict): 891 """ 892 This is set up to write out multiple harness even if only gnumake 893 is supported now 894 """ 895 eval("self.write_"+output+"(dataDict)") 896 return 897 898def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 899 if output is None: 900 output = 'gnumake' 901 902 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 903 if petsc_arch: 904 if len(petsc_arch.split(os.path.sep))>1: 905 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 906 907 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 908 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 909 testdir=testdir) 910 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 911 pEx.writeHarness(output,dataDict) 912 913if __name__ == '__main__': 914 import optparse 915 parser = optparse.OptionParser() 916 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 917 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 918 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 919 parser.add_option('--output', help='Location to write output file', default=None) 920 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 921 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 922 opts, extra_args = parser.parse_args() 923 opts, extra_args = parser.parse_args() 924 if extra_args: 925 import sys 926 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 927 exit(1) 928 main(petsc_arch=opts.petsc_arch, output=opts.output, verbose=opts.verbose, 929 single_ex=opts.single_executable, srcdir=opts.srcdir, testdir=opts.testdir) 930