1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 3 modes of running tests: Normal builds, test installs from builds, 23test installs from within install dir. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. Test immediately after build through makefile & lib/petsc/conf/test: 46 (Not in installDir, but using prefix dir. PETSC_ARCH='') 47 48 +---------------------+----------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+----------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+----------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+----------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_SRC_DIR/src | 56 +---------------------+----------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+----------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 60 +---------------------+----------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 62 +---------------------+----------------------------------+ 63 64Case 3. From install dir: 65 66 +---------------------+-------------------------------------------------------+ 67 | PETSC_DIR | <prefix dir> | 68 +---------------------+-------------------------------------------------------+ 69 | PETSC_ARCH | '' | 70 +---------------------+-------------------------------------------------------+ 71 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 72 +---------------------+-------------------------------------------------------+ 73 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 74 +---------------------+-------------------------------------------------------+ 75 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 76 +---------------------+-------------------------------------------------------+ 77 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 78 +---------------------+-------------------------------------------------------+ 79 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 80 +---------------------+-------------------------------------------------------+ 81 82""" 83class generateExamples(Petsc): 84 """ 85 gmakegen.py has basic structure for finding the files, writing out 86 the dependencies, etc. 87 """ 88 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 89 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 90 91 self.single_ex=single_ex 92 93 # Set locations to handle movement 94 self.inInstallDir=self.getInInstallDir(thisscriptdir) 95 96 if not self.inInstallDir: 97 if not petsc_arch == '': 98 # Case 1 discussed above 99 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 100 self.srcdir=os.path.join(self.petsc_dir,'src') 101 else: 102 # Case 2 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 105 else: 106 # Case 3 discussed above 107 # set PETSC_ARCH to install directory to allow script to work in both 108 dirlist=thisscriptdir.split(os.path.sep) 109 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 110 self.arch_dir=installdir 111 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 112 113 # Do some initialization 114 if testdir: 115 # If full path given, then use it, otherwise assume relative to arch_dir 116 if testdir.strip().startswith(os.path.sep): 117 self.testroot_dir=testdir.strip() 118 else: 119 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 120 else: 121 self.testroot_dir=os.path.join(self.arch_dir,"tests") 122 123 self.ptNaming=True 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64".split() 131 self.languages="fortran cuda cxx".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in PKGS: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def nameSpace(self,srcfile,srcdir): 181 """ 182 Because the scripts have a non-unique naming, the pretty-printing 183 needs to convey the srcdir and srcfile. There are two ways of doing this. 184 """ 185 if self.ptNaming: 186 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 187 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 188 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 189 nameString=prefix+srcfile 190 else: 191 #nameString=srcdir+": "+srcfile 192 nameString=srcfile 193 return nameString 194 195 def getLanguage(self,srcfile): 196 """ 197 Based on the source, determine associated language as found in gmakegen.LANGS 198 Can we just return srcext[1:\] now? 199 """ 200 langReq=None 201 srcext=os.path.splitext(srcfile)[-1] 202 if srcext in ".F90".split(): langReq="F90" 203 if srcext in ".F".split(): langReq="F" 204 if srcext in ".cxx".split(): langReq="cxx" 205 if srcext == ".cu": langReq="cu" 206 if srcext == ".c": langReq="c" 207 #if not langReq: print "ERROR: ", srcext, srcfile 208 return langReq 209 210 def _getLoopVars(self,inDict,testname, isSubtest=False): 211 """ 212 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 213 Return: 214 inDict['args']: -ksp_monitor 215 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 216 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 217 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 218 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 219 subst should be passed in instead of inDict 220 """ 221 loopVars={}; newargs="" 222 lkeys=inDict.keys() 223 lsuffix='_' 224 from testparse import parseLoopArgs 225 for key in lkeys: 226 if type(inDict[key])!=types.StringType: continue 227 keystr = str(inDict[key]) 228 akey=('subargs' if key=='args' else key) # what to assign 229 if akey not in inDict: inDict[akey]='' 230 varlist=[] 231 for varset in re.split('-(?=[a-zA-Z])',keystr): 232 if not varset.strip(): continue 233 if '{{' in varset: 234 keyvar,lvars,ftype=parseLoopArgs(varset) 235 if akey not in loopVars: loopVars[akey]={} 236 varlist.append(keyvar) 237 loopVars[akey][keyvar]=[keyvar,lvars] 238 if akey=='nsize': 239 inDict[akey] = '${' + keyvar + '}' 240 lsuffix+=akey+'-'+inDict[akey]+'_' 241 else: 242 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 243 lsuffix+=keyvar+'-${' + keyvar + '}_' 244 else: 245 if key=='args': newargs+=" -"+varset.strip() 246 if varlist: loopVars[akey]['varlist']=varlist 247 248 249 # For subtests, args are always substituted in (not top level) 250 if isSubtest: 251 inDict['subargs']+=" "+newargs.strip() 252 inDict['args']='' 253 if 'label_suffix' in inDict: 254 inDict['label_suffix']+=lsuffix.rstrip('_') 255 else: 256 inDict['label_suffix']=lsuffix.rstrip('_') 257 else: 258 if loopVars.keys(): 259 inDict['args']=newargs.strip() 260 inDict['label_suffix']=lsuffix.rstrip('_') 261 if loopVars.keys(): 262 return loopVars 263 else: 264 return None 265 266 def getArgLabel(self,testDict): 267 """ 268 In all of the arguments in the test dictionary, create a simple 269 string for searching within the makefile system. For simplicity in 270 search, remove "-", for strings, etc. 271 Also, concatenate the arg commands 272 For now, ignore nsize -- seems hard to search for anyway 273 """ 274 # Collect all of the args associated with a test 275 argStr=("" if 'args' not in testDict else testDict['args']) 276 if 'subtests' in testDict: 277 for stest in testDict["subtests"]: 278 sd=testDict[stest] 279 argStr=argStr+("" if 'args' not in sd else sd['args']) 280 281 # Now go through and cleanup 282 argStr=re.sub('{{(.*?)}}',"",argStr) 283 argStr=re.sub('-'," ",argStr) 284 for digit in string.digits: argStr=re.sub(digit," ",argStr) 285 argStr=re.sub("\.","",argStr) 286 argStr=re.sub(",","",argStr) 287 argStr=re.sub('\+',' ',argStr) 288 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 289 return argStr.strip() 290 291 def addToSources(self,exfile,root,srcDict): 292 """ 293 Put into data structure that allows easy generation of makefile 294 """ 295 rpath=self.srcrelpath(root) 296 pkg=rpath.split(os.path.sep)[1] 297 relpfile=os.path.join(rpath,exfile) 298 lang=self.getLanguage(exfile) 299 if not lang: return 300 self.sources[pkg][lang]['srcs'].append(relpfile) 301 self.sources[pkg][lang][relpfile] = [] 302 if 'depends' in srcDict: 303 depSrcList=srcDict['depends'].split() 304 for depSrc in depSrcList: 305 depObj=os.path.splitext(depSrc)[0]+".o" 306 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 307 308 # In gmakefile, ${TESTDIR} var specifies the object compilation 309 testsdir=self.srcrelpath(root)+"/" 310 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 311 self.objects[pkg].append(objfile) 312 return 313 314 def addToTests(self,test,root,exfile,execname,testDict): 315 """ 316 Put into data structure that allows easy generation of makefile 317 Organized by languages to allow testing of languages 318 """ 319 rpath=self.srcrelpath(root) 320 pkg=rpath.split("/")[1] 321 #nmtest=self.nameSpace(test,root) 322 nmtest=os.path.join(rpath,test) 323 lang=self.getLanguage(exfile) 324 if not lang: return 325 self.tests[pkg][lang][nmtest]={} 326 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 327 self.tests[pkg][lang][nmtest]['exec']=execname 328 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 329 return 330 331 def getExecname(self,exfile,root): 332 """ 333 Generate bash script using template found next to this file. 334 This file is read in at constructor time to avoid file I/O 335 """ 336 rpath=self.srcrelpath(root) 337 if self.single_ex: 338 execname=rpath.split("/")[1]+"-ex" 339 else: 340 execname=os.path.splitext(exfile)[0] 341 return execname 342 343 def getSubstVars(self,testDict,rpath,testname): 344 """ 345 Create a dictionary with all of the variables that get substituted 346 into the template commands found in example_template.py 347 """ 348 subst={} 349 350 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 351 if 'nsize' not in testDict: testDict['nsize']=1 352 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 353 for ak in testparse.acceptedkeys: 354 if ak=='test': continue 355 subst[ak]=(testDict[ak] if ak in testDict else '') 356 357 # Now do other variables 358 subst['execname']=testDict['execname'] 359 if 'filter' in testDict: 360 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 361 362 # Others 363 subst['subargs']='' # Default. For variables override 364 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 365 subst['label_suffix']='' 366 subst['comments']="\n#".join(subst['comments'].split("\n")) 367 if subst['comments']: subst['comments']="#"+subst['comments'] 368 subst['exec']="../"+subst['execname'] 369 subst['testroot']=self.testroot_dir 370 subst['testname']=testname 371 dp = self.conf.get('DATAFILESPATH','') 372 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 373 374 # This is used to label some matrices 375 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 376 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 377 378 # These can have for loops and are treated separately later 379 subst['nsize']=str(subst['nsize']) 380 381 #Conf vars 382 if self.petsc_arch.find('valgrind')>=0: 383 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 384 else: 385 subst['mpiexec']=self.conf['MPIEXEC'] 386 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 387 subst['petsc_arch']=self.petsc_arch 388 if not self.inInstallDir: 389 if not self.petsc_arch == '': 390 # Case 1 391 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 392 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 393 else: 394 # Case 2 395 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 396 subst['PETSC_BINDIR']=os.path.join(os.path.dirname(self.srcdir),'lib','petsc','bin') 397 else: 398 # Case 3 399 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 400 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 401 subst['diff']=self.conf['DIFF'] 402 subst['rm']=self.conf['RM'] 403 subst['grep']=self.conf['GREP'] 404 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 405 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 406 407 # Output file is special because of subtests override 408 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 409 if not "_" in defroot: defroot=defroot+"_1" 410 subst['defroot']=defroot 411 subst['label']=self.nameSpace(defroot,subst['srcdir']) 412 subst['redirect_file']=defroot+".tmp" 413 if 'output_file' not in testDict: 414 subst['output_file']="output/"+defroot+".out" 415 # Add in the full path here. 416 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 417 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 418 if not subst['TODO']: 419 print "Warning: "+subst['output_file']+" not found." 420 # Worry about alt files here -- see 421 # src/snes/examples/tutorials/output/ex22*.out 422 altlist=[subst['output_file']] 423 basefile,ext = os.path.splitext(subst['output_file']) 424 for i in range(1,9): 425 altroot=basefile+"_alt" 426 if i > 1: altroot=altroot+"_"+str(i) 427 af=altroot+".out" 428 srcaf=os.path.join(subst['srcdir'],af) 429 fullaf=os.path.join(self.petsc_dir,srcaf) 430 if os.path.isfile(fullaf): altlist.append(srcaf) 431 if len(altlist)>1: subst['altfiles']=altlist 432 #if len(altlist)>1: print "Found alt files: ",altlist 433 434 return subst 435 436 def getCmds(self,subst,i): 437 """ 438 Generate bash script using template found next to this file. 439 This file is read in at constructor time to avoid file I/O 440 """ 441 nindnt=i # the start and has to be consistent with below 442 cmdindnt=self.indent*nindnt 443 cmdLines="" 444 445 # MPI is the default -- but we have a few odd commands 446 if not subst['command']: 447 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 448 else: 449 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 450 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 451 452 cmdLines+=cmdindnt+'if test $res = 0; then\n' 453 diffindnt=self.indent*(nindnt+1) 454 if not subst['filter_output']: 455 if 'altfiles' not in subst: 456 cmd=diffindnt+self._substVars(subst,example_template.difftest) 457 else: 458 # Have to do it by hand a bit because of variable number of alt files 459 rf=subst['redirect_file'] 460 cmd=diffindnt+example_template.difftest.split('@')[0] 461 for i in range(len(subst['altfiles'])): 462 af=subst['altfiles'][i] 463 cmd+=af+' '+rf 464 if i!=len(subst['altfiles'])-1: 465 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 466 cmd+=' || ${diff_exe} ' 467 else: 468 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 469 cmd+=subst['label_suffix']+' ""' # Quotes are painful 470 else: 471 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 472 cmdLines+=cmd+"\n" 473 cmdLines+=cmdindnt+'else\n' 474 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 475 cmdLines+=cmdindnt+'fi\n' 476 return cmdLines 477 478 def _substVars(self,subst,origStr): 479 """ 480 Substitute variables 481 """ 482 Str=origStr 483 for subkey in subst: 484 if type(subst[subkey])!=types.StringType: continue 485 patt="@"+subkey.upper()+"@" 486 Str=re.sub(patt,subst[subkey],Str) 487 return Str 488 489 def _writeTodoSkip(self,fh,tors,reasons,footer): 490 """ 491 Write out the TODO and SKIP lines in the file 492 The TODO or SKIP variable, tors, should be lower case 493 """ 494 TORS=tors.upper() 495 template=eval("example_template."+tors+"line") 496 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 497 tab = '' 498 if reasons: 499 fh.write('if ! $force; then\n') 500 tab = tab + ' ' 501 if reasons == ["Requires DATAFILESPATH"]: 502 # The only reason not to run is DATAFILESPATH, which we check at run-time 503 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 504 tab = tab + ' ' 505 if reasons: 506 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 507 fh.write(tab+footer+"\n") 508 fh.write(tab+"exit\n") 509 if reasons == ["Requires DATAFILESPATH"]: 510 fh.write(' fi\n') 511 if reasons: 512 fh.write('fi\n') 513 fh.write('\n\n') 514 return 515 516 def getLoopVarsHead(self,loopVars,i): 517 """ 518 Generate a nicely indented string with the format loops 519 Here is what the data structure looks like 520 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 521 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 522 loopVars['subargs']['pc_type']=["j","cholesky sor"] 523 """ 524 outstr=''; indnt=self.indent 525 for key in loopVars: 526 for var in loopVars[key]['varlist']: 527 varval=loopVars[key][var] 528 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 529 i = i + 1 530 return (outstr,i) 531 532 def getLoopVarsFoot(self,loopVars,i): 533 outstr=''; indnt=self.indent 534 for key in loopVars: 535 for var in loopVars[key]['varlist']: 536 i = i - 1 537 outstr += indnt * i + "done\n" 538 return (outstr,i) 539 540 def genRunScript(self,testname,root,isRun,srcDict): 541 """ 542 Generate bash script using template found next to this file. 543 This file is read in at constructor time to avoid file I/O 544 """ 545 # runscript_dir directory has to be consistent with gmakefile 546 testDict=srcDict[testname] 547 rpath=self.srcrelpath(root) 548 runscript_dir=os.path.join(self.testroot_dir,rpath) 549 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 550 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 551 552 # Get variables to go into shell scripts. last time testDict used 553 subst=self.getSubstVars(testDict,rpath,testname) 554 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 555 #if '33_' in testname: print subst['subargs'] 556 557 #Handle runfiles 558 for lfile in subst.get('localrunfiles','').split(): 559 fullfile=os.path.join(root,lfile) 560 if os.path.isdir(fullfile): 561 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 562 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 563 else: 564 shutil.copy(fullfile,runscript_dir) 565 # Check subtests for local runfiles 566 for stest in subst.get("subtests",[]): 567 for lfile in testDict[stest].get('localrunfiles','').split(): 568 fullfile=os.path.join(root,lfile) 569 if os.path.isdir(fullfile): 570 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 571 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 572 else: 573 shutil.copy(fullfile,self.runscript_dir) 574 575 # Now substitute the key variables into the header and footer 576 header=self._substVars(subst,example_template.header) 577 # The header is done twice to enable @...@ in header 578 header=self._substVars(subst,header) 579 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 580 581 # Start writing the file 582 fh.write(header+"\n") 583 584 # If there is a TODO or a SKIP then we do it before writing out the 585 # rest of the command (which is useful for working on the test) 586 # SKIP and TODO can be for the source file or for the runs 587 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 588 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 589 590 j=0 # for indentation 591 592 if loopVars: 593 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 594 if (loopHead): fh.write(loopHead+"\n") 595 596 # Subtests are special 597 if 'subtests' in testDict: 598 substP=subst # Subtests can inherit args but be careful 599 k=0 # for label suffixes 600 for stest in testDict["subtests"]: 601 subst=substP.copy() 602 subst.update(testDict[stest]) 603 # nsize is special because it is usually overwritten 604 if 'nsize' in testDict[stest]: 605 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 606 else: 607 fh.write("nsize=1\n") 608 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 609 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 610 #if '10_9' in testname: print sLoopVars 611 if sLoopVars: 612 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 613 fh.write(sLoopHead+"\n") 614 fh.write(self.getCmds(subst,j)+"\n") 615 if sLoopVars: 616 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 617 fh.write(sLoopFoot+"\n") 618 else: 619 fh.write(self.getCmds(subst,j)+"\n") 620 621 if loopVars: 622 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 623 fh.write(loopFoot+"\n") 624 625 fh.write(footer+"\n") 626 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 627 #if '10_9' in testname: sys.exit() 628 return 629 630 def genScriptsAndInfo(self,exfile,root,srcDict): 631 """ 632 Generate scripts from the source file, determine if built, etc. 633 For every test in the exfile with info in the srcDict: 634 1. Determine if it needs to be run for this arch 635 2. Generate the script 636 3. Generate the data needed to write out the makefile in a 637 convenient way 638 All tests are *always* run, but some may be SKIP'd per the TAP standard 639 """ 640 debug=False 641 execname=self.getExecname(exfile,root) 642 isBuilt=self._isBuilt(exfile,srcDict) 643 for test in srcDict: 644 if test in self.buildkeys: continue 645 if debug: print self.nameSpace(exfile,root), test 646 srcDict[test]['execname']=execname # Convenience in generating scripts 647 isRun=self._isRun(srcDict[test]) 648 self.genRunScript(test,root,isRun,srcDict) 649 srcDict[test]['isrun']=isRun 650 self.addToTests(test,root,exfile,execname,srcDict[test]) 651 652 # This adds to datastructure for building deps 653 if isBuilt: self.addToSources(exfile,root,srcDict) 654 return 655 656 def _isBuilt(self,exfile,srcDict): 657 """ 658 Determine if this file should be built. 659 """ 660 # Get the language based on file extension 661 srcDict['SKIP'] = [] 662 lang=self.getLanguage(exfile) 663 if (lang=="F" or lang=="F90"): 664 if not self.have_fortran: 665 srcDict["SKIP"].append("Fortran required for this test") 666 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 667 srcDict["SKIP"].append("Fortran f90freeform required for this test") 668 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 669 srcDict["SKIP"].append("CUDA required for this test") 670 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 671 srcDict["SKIP"].append("C++ required for this test") 672 673 # Deprecated source files 674 if srcDict.get("TODO"): 675 return False 676 677 # isRun can work with srcDict to handle the requires 678 if "requires" in srcDict: 679 if srcDict["requires"]: 680 return self._isRun(srcDict) 681 682 return srcDict['SKIP'] == [] 683 684 685 def _isRun(self,testDict, debug=False): 686 """ 687 Based on the requirements listed in the src file and the petscconf.h 688 info, determine whether this test should be run or not. 689 """ 690 indent=" " 691 692 if 'SKIP' not in testDict: 693 testDict['SKIP'] = [] 694 # MPI requirements 695 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 696 if debug: print indent+"Cannot run parallel tests" 697 testDict['SKIP'].append("Parallel test with serial build") 698 699 # The requirements for the test are the sum of all the run subtests 700 if 'subtests' in testDict: 701 if 'requires' not in testDict: testDict['requires']="" 702 for stest in testDict['subtests']: 703 if 'requires' in testDict[stest]: 704 testDict['requires']+=" "+testDict[stest]['requires'] 705 if 'nsize' in testDict[stest]: 706 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 707 testDict['SKIP'].append("Parallel test with serial build") 708 709 710 # Now go through all requirements 711 if 'requires' in testDict: 712 for requirement in testDict['requires'].split(): 713 requirement=requirement.strip() 714 if not requirement: continue 715 if debug: print indent+"Requirement: ", requirement 716 isNull=False 717 if requirement.startswith("!"): 718 requirement=requirement[1:]; isNull=True 719 # Precision requirement for reals 720 if requirement in self.precision_types: 721 if self.conf['PETSC_PRECISION']==requirement: 722 if isNull: 723 testDict['SKIP'].append("not "+requirement+" required") 724 continue 725 continue # Success 726 elif not isNull: 727 testDict['SKIP'].append(requirement+" required") 728 continue 729 # Precision requirement for ints 730 if requirement in self.integer_types: 731 if requirement=="int32": 732 if self.conf['PETSC_SIZEOF_INT']==4: 733 if isNull: 734 testDict['SKIP'].append("not int32 required") 735 continue 736 continue # Success 737 elif not isNull: 738 testDict['SKIP'].append("int32 required") 739 continue 740 if requirement=="int64": 741 if self.conf['PETSC_SIZEOF_INT']==8: 742 if isNull: 743 testDict['SKIP'].append("NOT int64 required") 744 continue 745 continue # Success 746 elif not isNull: 747 testDict['SKIP'].append("int64 required") 748 continue 749 # Datafilespath 750 if requirement=="datafilespath" and not isNull: 751 testDict['SKIP'].append("Requires DATAFILESPATH") 752 continue 753 # Defines -- not sure I have comments matching 754 if "define(" in requirement.lower(): 755 reqdef=requirement.split("(")[1].split(")")[0] 756 if reqdef in self.conf: 757 if isNull: 758 testDict['SKIP'].append("Null requirement not met: "+requirement) 759 continue 760 continue # Success 761 elif not isNull: 762 testDict['SKIP'].append("Required: "+requirement) 763 continue 764 765 # Rest should be packages that we can just get from conf 766 if requirement == "complex": 767 petscconfvar="PETSC_USE_COMPLEX" 768 else: 769 petscconfvar="PETSC_HAVE_"+requirement.upper() 770 if self.conf.get(petscconfvar): 771 if isNull: 772 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 773 continue 774 continue # Success 775 elif not isNull: 776 if debug: print "requirement not found: ", requirement 777 testDict['SKIP'].append(petscconfvar+" requirement not met") 778 continue 779 780 return testDict['SKIP'] == [] 781 782 def genPetscTests_summarize(self,dataDict): 783 """ 784 Required method to state what happened 785 """ 786 if not self.summarize: return 787 indent=" " 788 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 789 fh=open(fhname,"w") 790 #print "See ", fhname 791 for root in dataDict: 792 relroot=self.srcrelpath(root) 793 pkg=relroot.split("/")[1] 794 fh.write(relroot+"\n") 795 allSrcs=[] 796 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 797 for exfile in dataDict[root]: 798 # Basic information 799 rfile=os.path.join(relroot,exfile) 800 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 801 fh.write(indent+exfile+indent*4+builtStatus+"\n") 802 803 for test in dataDict[root][exfile]: 804 if test in self.buildkeys: continue 805 line=indent*2+test 806 fh.write(line+"\n") 807 # Looks nice to have the keys in order 808 #for key in dataDict[root][exfile][test]: 809 for key in "isrun abstracted nsize args requires script".split(): 810 if key not in dataDict[root][exfile][test]: continue 811 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 812 fh.write(line+"\n") 813 fh.write("\n") 814 fh.write("\n") 815 fh.write("\n") 816 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 817 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 818 fh.close() 819 return 820 821 def genPetscTests(self,root,dirs,files,dataDict): 822 """ 823 Go through and parse the source files in the directory to generate 824 the examples based on the metadata contained in the source files 825 """ 826 debug=False 827 # Use examplesAnalyze to get what the makefles think are sources 828 #self.examplesAnalyze(root,dirs,files,anlzDict) 829 830 dataDict[root]={} 831 832 for exfile in files: 833 #TST: Until we replace files, still leaving the orginals as is 834 #if not exfile.startswith("new_"+"ex"): continue 835 #if not exfile.startswith("ex"): continue 836 837 # Ignore emacs and other temporary files 838 if exfile.startswith("."): continue 839 if exfile.startswith("#"): continue 840 841 # Convenience 842 fullex=os.path.join(root,exfile) 843 if self.verbose: print(' --> '+fullex) 844 dataDict[root].update(testparse.parseTestFile(fullex,0)) 845 if exfile in dataDict[root]: 846 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 847 848 return 849 850 def walktree(self,top): 851 """ 852 Walk a directory tree, starting from 'top' 853 """ 854 #print "action", action 855 # Goal of action is to fill this dictionary 856 dataDict={} 857 for root, dirs, files in os.walk(top, topdown=True): 858 if not "examples" in root: continue 859 if "dSYM" in root: continue 860 if os.path.basename(root.rstrip("/")) == 'output': continue 861 if self.verbose: print(root) 862 self.genPetscTests(root,dirs,files,dataDict) 863 # Now summarize this dictionary 864 self.genPetscTests_summarize(dataDict) 865 return dataDict 866 867 def gen_gnumake(self, fd): 868 """ 869 Overwrite of the method in the base PETSc class 870 """ 871 def write(stem, srcs): 872 for lang in LANGS: 873 if srcs[lang]['srcs']: 874 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 875 for pkg in PKGS: 876 srcs = self.gen_pkg(pkg) 877 write('testsrcs-' + pkg, srcs) 878 # Handle dependencies 879 for lang in LANGS: 880 for exfile in srcs[lang]['srcs']: 881 if exfile in srcs[lang]: 882 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 883 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 884 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 885 if deps: 886 # The executable literally depends on the object file because it is linked 887 fd.write(ex +": " + " ".join(deps) +'\n') 888 # The object file containing 'main' does not normally depend on other object 889 # files, but it does when it includes their modules. This dependency is 890 # overly blunt and could be reduced to only depend on object files for 891 # modules that are used, like "*f90aux.o". 892 fd.write(exfo +": " + " ".join(deps) +'\n') 893 894 return self.gendeps 895 896 def gen_pkg(self, pkg): 897 """ 898 Overwrite of the method in the base PETSc class 899 """ 900 return self.sources[pkg] 901 902 def write_gnumake(self,dataDict): 903 """ 904 Write out something similar to files from gmakegen.py 905 906 Test depends on script which also depends on source 907 file, but since I don't have a good way generating 908 acting on a single file (oops) just depend on 909 executable which in turn will depend on src file 910 """ 911 # Different options for how to set up the targets 912 compileExecsFirst=False 913 914 # Open file 915 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 916 fd = open(arch_files, 'w') 917 918 # Write out the sources 919 gendeps = self.gen_gnumake(fd) 920 921 # Write out the tests and execname targets 922 fd.write("\n#Tests and executables\n") # Delimiter 923 924 for pkg in PKGS: 925 # These grab the ones that are built 926 for lang in LANGS: 927 testdeps=[] 928 for ftest in self.tests[pkg][lang]: 929 test=os.path.basename(ftest) 930 basedir=os.path.dirname(ftest) 931 testdeps.append(self.nameSpace(test,basedir)) 932 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 933 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 934 935 # test targets 936 for ftest in self.tests[pkg][lang]: 937 test=os.path.basename(ftest) 938 basedir=os.path.dirname(ftest) 939 testdir="${TESTDIR}/"+basedir+"/" 940 nmtest=self.nameSpace(test,basedir) 941 rundir=os.path.join(testdir,test) 942 #print test, nmtest 943 script=test+".sh" 944 945 # Deps 946 exfile=self.tests[pkg][lang][ftest]['exfile'] 947 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 948 localexec=self.tests[pkg][lang][ftest]['exec'] 949 execname=os.path.join(testdir,localexec) 950 fullscript=os.path.join(testdir,script) 951 tmpfile=os.path.join(testdir,test,test+".tmp") 952 953 # *.counts depends on the script and either executable (will 954 # be run) or the example source file (SKIP or TODO) 955 fd.write('%s.counts : %s %s' 956 % (os.path.join('$(TESTDIR)/counts', nmtest), 957 fullscript, 958 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 959 ) 960 if exfile in self.sources[pkg][lang]: 961 for dep in self.sources[pkg][lang][exfile]: 962 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 963 fd.write('\n') 964 965 # Now write the args: 966 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 967 968 fd.close() 969 return 970 971 def writeHarness(self,output,dataDict): 972 """ 973 This is set up to write out multiple harness even if only gnumake 974 is supported now 975 """ 976 eval("self.write_"+output+"(dataDict)") 977 return 978 979def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 980 if output is None: 981 output = 'gnumake' 982 983 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 984 if petsc_arch: 985 if len(petsc_arch.split(os.path.sep))>1: 986 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 987 988 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 989 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 990 testdir=testdir) 991 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 992 pEx.writeHarness(output,dataDict) 993 994if __name__ == '__main__': 995 import optparse 996 parser = optparse.OptionParser() 997 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 998 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 999 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1000 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1001 parser.add_option('--output', help='Location to write output file', default=None) 1002 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1003 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 1004 1005 opts, extra_args = parser.parse_args() 1006 if extra_args: 1007 import sys 1008 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1009 exit(1) 1010 1011 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1012 output=opts.output, verbose=opts.verbose, 1013 single_ex=opts.single_executable, srcdir=opts.srcdir, 1014 testdir=opts.testdir) 1015