1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 3 modes of running tests: Normal builds, test installs from builds, 23test installs from within install dir. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. Test immediately after build through makefile & lib/petsc/conf/test: 46 (Not in installDir, but using prefix dir. PETSC_ARCH='') 47 48 +---------------------+----------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+----------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+----------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+----------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_SRC_DIR/src | 56 +---------------------+----------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+----------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 60 +---------------------+----------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 62 +---------------------+----------------------------------+ 63 64Case 3. From install dir: 65 66 +---------------------+-------------------------------------------------------+ 67 | PETSC_DIR | <prefix dir> | 68 +---------------------+-------------------------------------------------------+ 69 | PETSC_ARCH | '' | 70 +---------------------+-------------------------------------------------------+ 71 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 72 +---------------------+-------------------------------------------------------+ 73 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 74 +---------------------+-------------------------------------------------------+ 75 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 76 +---------------------+-------------------------------------------------------+ 77 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 78 +---------------------+-------------------------------------------------------+ 79 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 80 +---------------------+-------------------------------------------------------+ 81 82""" 83class generateExamples(Petsc): 84 """ 85 gmakegen.py has basic structure for finding the files, writing out 86 the dependencies, etc. 87 """ 88 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 89 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 90 91 self.single_ex=single_ex 92 93 # Set locations to handle movement 94 self.inInstallDir=self.getInInstallDir(thisscriptdir) 95 96 if not self.inInstallDir: 97 if not petsc_arch == '': 98 # Case 1 discussed above 99 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 100 self.srcdir=os.path.join(self.petsc_dir,'src') 101 else: 102 # Case 2 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 105 else: 106 # Case 3 discussed above 107 # set PETSC_ARCH to install directory to allow script to work in both 108 dirlist=thisscriptdir.split(os.path.sep) 109 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 110 self.arch_dir=installdir 111 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 112 113 # Do some initialization 114 if testdir: 115 # If full path given, then use it, otherwise assume relative to arch_dir 116 if testdir.strip().startswith(os.path.sep): 117 self.testroot_dir=testdir.strip() 118 else: 119 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 120 else: 121 self.testroot_dir=os.path.join(self.arch_dir,"tests") 122 123 self.ptNaming=True 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64".split() 131 self.languages="fortran cuda cxx".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in PKGS: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def nameSpace(self,srcfile,srcdir): 181 """ 182 Because the scripts have a non-unique naming, the pretty-printing 183 needs to convey the srcdir and srcfile. There are two ways of doing this. 184 """ 185 if self.ptNaming: 186 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 187 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 188 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 189 nameString=prefix+srcfile 190 else: 191 #nameString=srcdir+": "+srcfile 192 nameString=srcfile 193 return nameString 194 195 def getLanguage(self,srcfile): 196 """ 197 Based on the source, determine associated language as found in gmakegen.LANGS 198 Can we just return srcext[1:\] now? 199 """ 200 langReq=None 201 srcext=os.path.splitext(srcfile)[-1] 202 if srcext in ".F90".split(): langReq="F90" 203 if srcext in ".F".split(): langReq="F" 204 if srcext in ".cxx".split(): langReq="cxx" 205 if srcext == ".cu": langReq="cu" 206 if srcext == ".c": langReq="c" 207 #if not langReq: print "ERROR: ", srcext, srcfile 208 return langReq 209 210 def _getLoopVars(self,inDict,testname, isSubtest=False): 211 """ 212 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 213 Return: 214 inDict['args']: -ksp_monitor 215 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 216 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 217 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 218 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 219 subst should be passed in instead of inDict 220 """ 221 loopVars={}; newargs="" 222 lkeys=inDict.keys() 223 lsuffix='_' 224 from testparse import parseLoopArgs 225 for key in lkeys: 226 if type(inDict[key])!=types.StringType: continue 227 keystr = str(inDict[key]) 228 akey=('subargs' if key=='args' else key) # what to assign 229 if akey not in inDict: inDict[akey]='' 230 varlist=[] 231 for varset in re.split('-(?=[a-zA-Z])',keystr): 232 if not varset.strip(): continue 233 if '{{' in varset: 234 keyvar,lvars,ftype=parseLoopArgs(varset) 235 if akey not in loopVars: loopVars[akey]={} 236 varlist.append(keyvar) 237 loopVars[akey][keyvar]=[keyvar,lvars] 238 if akey=='nsize': 239 inDict[akey] = '${' + keyvar + '}' 240 lsuffix+=akey+'-'+inDict[akey]+'_' 241 else: 242 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 243 lsuffix+=keyvar+'-${' + keyvar + '}_' 244 else: 245 if key=='args': newargs+=" -"+varset.strip() 246 if varlist: loopVars[akey]['varlist']=varlist 247 248 249 # For subtests, args are always substituted in (not top level) 250 if isSubtest: 251 inDict['subargs']+=" "+newargs.strip() 252 inDict['args']='' 253 if 'label_suffix' in inDict: 254 inDict['label_suffix']+=lsuffix.rstrip('_') 255 else: 256 inDict['label_suffix']=lsuffix.rstrip('_') 257 else: 258 if loopVars.keys(): 259 inDict['args']=newargs.strip() 260 inDict['label_suffix']=lsuffix.rstrip('_') 261 if loopVars.keys(): 262 return loopVars 263 else: 264 return None 265 266 def getArgLabel(self,testDict): 267 """ 268 In all of the arguments in the test dictionary, create a simple 269 string for searching within the makefile system. For simplicity in 270 search, remove "-", for strings, etc. 271 Also, concatenate the arg commands 272 For now, ignore nsize -- seems hard to search for anyway 273 """ 274 # Collect all of the args associated with a test 275 argStr=("" if 'args' not in testDict else testDict['args']) 276 if 'subtests' in testDict: 277 for stest in testDict["subtests"]: 278 sd=testDict[stest] 279 argStr=argStr+("" if 'args' not in sd else sd['args']) 280 281 # Now go through and cleanup 282 argStr=re.sub('{{(.*?)}}',"",argStr) 283 argStr=re.sub('-'," ",argStr) 284 for digit in string.digits: argStr=re.sub(digit," ",argStr) 285 argStr=re.sub("\.","",argStr) 286 argStr=re.sub(",","",argStr) 287 argStr=re.sub('\+',' ',argStr) 288 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 289 return argStr.strip() 290 291 def addToSources(self,exfile,root,srcDict): 292 """ 293 Put into data structure that allows easy generation of makefile 294 """ 295 rpath=self.srcrelpath(root) 296 pkg=rpath.split(os.path.sep)[1] 297 relpfile=os.path.join(rpath,exfile) 298 lang=self.getLanguage(exfile) 299 if not lang: return 300 self.sources[pkg][lang]['srcs'].append(relpfile) 301 self.sources[pkg][lang][relpfile] = [] 302 if 'depends' in srcDict: 303 depSrcList=srcDict['depends'].split() 304 for depSrc in depSrcList: 305 depObj=os.path.splitext(depSrc)[0]+".o" 306 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 307 308 # In gmakefile, ${TESTDIR} var specifies the object compilation 309 testsdir=self.srcrelpath(root)+"/" 310 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 311 self.objects[pkg].append(objfile) 312 return 313 314 def addToTests(self,test,root,exfile,execname,testDict): 315 """ 316 Put into data structure that allows easy generation of makefile 317 Organized by languages to allow testing of languages 318 """ 319 rpath=self.srcrelpath(root) 320 pkg=rpath.split("/")[1] 321 #nmtest=self.nameSpace(test,root) 322 nmtest=os.path.join(rpath,test) 323 lang=self.getLanguage(exfile) 324 if not lang: return 325 self.tests[pkg][lang][nmtest]={} 326 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 327 self.tests[pkg][lang][nmtest]['exec']=execname 328 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 329 return 330 331 def getExecname(self,exfile,root): 332 """ 333 Generate bash script using template found next to this file. 334 This file is read in at constructor time to avoid file I/O 335 """ 336 rpath=self.srcrelpath(root) 337 if self.single_ex: 338 execname=rpath.split("/")[1]+"-ex" 339 else: 340 execname=os.path.splitext(exfile)[0] 341 return execname 342 343 def getSubstVars(self,testDict,rpath,testname): 344 """ 345 Create a dictionary with all of the variables that get substituted 346 into the template commands found in example_template.py 347 """ 348 subst={} 349 350 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 351 if 'nsize' not in testDict: testDict['nsize']=1 352 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 353 for ak in testparse.acceptedkeys: 354 if ak=='test': continue 355 subst[ak]=(testDict[ak] if ak in testDict else '') 356 357 # Now do other variables 358 subst['execname']=testDict['execname'] 359 if 'filter' in testDict: 360 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 361 362 # Others 363 subst['subargs']='' # Default. For variables override 364 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 365 subst['label_suffix']='' 366 subst['comments']="\n#".join(subst['comments'].split("\n")) 367 if subst['comments']: subst['comments']="#"+subst['comments'] 368 subst['exec']="../"+subst['execname'] 369 subst['testroot']=self.testroot_dir 370 subst['testname']=testname 371 dp = self.conf.get('DATAFILESPATH','') 372 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 373 374 # This is used to label some matrices 375 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 376 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 377 378 # These can have for loops and are treated separately later 379 subst['nsize']=str(subst['nsize']) 380 381 #Conf vars 382 if self.petsc_arch.find('valgrind')>=0: 383 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 384 else: 385 subst['mpiexec']=self.conf['MPIEXEC'] 386 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 387 subst['petsc_arch']=self.petsc_arch 388 if not self.inInstallDir: 389 if not self.petsc_arch == '': 390 # Case 1 391 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 392 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'bin') 393 else: 394 # Case 2 395 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 396 subst['PETSC_BINDIR']=os.path.join(os.path.dirname(self.srcdir),'bin') 397 else: 398 # Case 3 399 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 400 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 401 subst['diff']=self.conf['DIFF'] 402 subst['rm']=self.conf['RM'] 403 subst['grep']=self.conf['GREP'] 404 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 405 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 406 407 # Output file is special because of subtests override 408 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 409 if not "_" in defroot: defroot=defroot+"_1" 410 subst['defroot']=defroot 411 subst['label']=self.nameSpace(defroot,subst['srcdir']) 412 subst['redirect_file']=defroot+".tmp" 413 if 'output_file' not in testDict: 414 subst['output_file']="output/"+defroot+".out" 415 # Add in the full path here. 416 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 417 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 418 if not subst['TODO']: 419 print "Warning: "+subst['output_file']+" not found." 420 # Worry about alt files here -- see 421 # src/snes/examples/tutorials/output/ex22*.out 422 altlist=[subst['output_file']] 423 basefile,ext = os.path.splitext(subst['output_file']) 424 for i in range(1,9): 425 altroot=basefile+"_alt" 426 if i > 1: altroot=altroot+"_"+str(i) 427 af=altroot+".out" 428 srcaf=os.path.join(subst['srcdir'],af) 429 fullaf=os.path.join(self.petsc_dir,srcaf) 430 if os.path.isfile(fullaf): altlist.append(srcaf) 431 if len(altlist)>1: subst['altfiles']=altlist 432 #if len(altlist)>1: print "Found alt files: ",altlist 433 434 return subst 435 436 def getCmds(self,subst,i): 437 """ 438 Generate bash script using template found next to this file. 439 This file is read in at constructor time to avoid file I/O 440 """ 441 nindnt=i # the start and has to be consistent with below 442 cmdindnt=self.indent*nindnt 443 cmdLines="" 444 445 # MPI is the default -- but we have a few odd commands 446 if not subst['command']: 447 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 448 else: 449 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 450 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 451 452 cmdLines+=cmdindnt+'if test $res = 0; then\n' 453 diffindnt=self.indent*(nindnt+1) 454 if not subst['filter_output']: 455 if 'altfiles' not in subst: 456 cmd=diffindnt+self._substVars(subst,example_template.difftest) 457 else: 458 # Have to do it by hand a bit because of variable number of alt files 459 rf=subst['redirect_file'] 460 cmd=diffindnt+example_template.difftest.split('@')[0] 461 for i in range(len(subst['altfiles'])): 462 af=subst['altfiles'][i] 463 cmd+=af+' '+rf 464 if i!=len(subst['altfiles'])-1: 465 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 466 cmd+=' || ${diff_exe} ' 467 else: 468 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 469 cmd+=subst['label_suffix']+' ""' # Quotes are painful 470 else: 471 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 472 cmdLines+=cmd+"\n" 473 cmdLines+=cmdindnt+'else\n' 474 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 475 cmdLines+=cmdindnt+'fi\n' 476 return cmdLines 477 478 def _substVars(self,subst,origStr): 479 """ 480 Substitute variables 481 """ 482 Str=origStr 483 for subkey in subst: 484 if type(subst[subkey])!=types.StringType: continue 485 patt="@"+subkey.upper()+"@" 486 Str=re.sub(patt,subst[subkey],Str) 487 return Str 488 489 def _writeTodoSkip(self,fh,tors,reasons,footer): 490 """ 491 Write out the TODO and SKIP lines in the file 492 The TODO or SKIP variable, tors, should be lower case 493 """ 494 TORS=tors.upper() 495 template=eval("example_template."+tors+"line") 496 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 497 tab = '' 498 if reasons: 499 fh.write('if ! $force; then\n') 500 tab = tab + ' ' 501 if reasons == ["Requires DATAFILESPATH"]: 502 # The only reason not to run is DATAFILESPATH, which we check at run-time 503 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 504 tab = tab + ' ' 505 if reasons: 506 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 507 fh.write(tab+footer+"\n") 508 fh.write(tab+"exit\n") 509 if reasons == ["Requires DATAFILESPATH"]: 510 fh.write(' fi\n') 511 if reasons: 512 fh.write('fi\n') 513 fh.write('\n\n') 514 return 515 516 def getLoopVarsHead(self,loopVars,i): 517 """ 518 Generate a nicely indented string with the format loops 519 Here is what the data structure looks like 520 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 521 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 522 loopVars['subargs']['pc_type']=["j","cholesky sor"] 523 """ 524 outstr=''; indnt=self.indent 525 for key in loopVars: 526 for var in loopVars[key]['varlist']: 527 varval=loopVars[key][var] 528 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 529 i = i + 1 530 return (outstr,i) 531 532 def getLoopVarsFoot(self,loopVars,i): 533 outstr=''; indnt=self.indent 534 for key in loopVars: 535 for var in loopVars[key]['varlist']: 536 i = i - 1 537 outstr += indnt * i + "done\n" 538 return (outstr,i) 539 540 def genRunScript(self,testname,root,isRun,srcDict): 541 """ 542 Generate bash script using template found next to this file. 543 This file is read in at constructor time to avoid file I/O 544 """ 545 # runscript_dir directory has to be consistent with gmakefile 546 testDict=srcDict[testname] 547 rpath=self.srcrelpath(root) 548 runscript_dir=os.path.join(self.testroot_dir,rpath) 549 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 550 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 551 552 # Get variables to go into shell scripts. last time testDict used 553 subst=self.getSubstVars(testDict,rpath,testname) 554 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 555 #if '33_' in testname: print subst['subargs'] 556 557 #Handle runfiles 558 for lfile in subst.get('localrunfiles','').split(): 559 fullfile=os.path.join(root,lfile) 560 if os.path.isdir(fullfile): 561 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 562 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 563 else: 564 runscriptsub_dir=os.path.join(runscript_dir,os.path.dirname(lfile)) 565 if not os.path.isdir(runscriptsub_dir): os.makedirs(runscriptsub_dir) 566 shutil.copy(fullfile,runscriptsub_dir) 567 # Check subtests for local runfiles 568 for stest in subst.get("subtests",[]): 569 for lfile in testDict[stest].get('localrunfiles','').split(): 570 fullfile=os.path.join(root,lfile) 571 if os.path.isdir(fullfile): 572 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 573 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 574 else: 575 shutil.copy(fullfile,self.runscript_dir) 576 577 # Now substitute the key variables into the header and footer 578 header=self._substVars(subst,example_template.header) 579 # The header is done twice to enable @...@ in header 580 header=self._substVars(subst,header) 581 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 582 583 # Start writing the file 584 fh.write(header+"\n") 585 586 # If there is a TODO or a SKIP then we do it before writing out the 587 # rest of the command (which is useful for working on the test) 588 # SKIP and TODO can be for the source file or for the runs 589 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 590 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 591 592 j=0 # for indentation 593 594 if loopVars: 595 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 596 if (loopHead): fh.write(loopHead+"\n") 597 598 # Subtests are special 599 if 'subtests' in testDict: 600 substP=subst # Subtests can inherit args but be careful 601 k=0 # for label suffixes 602 for stest in testDict["subtests"]: 603 subst=substP.copy() 604 subst.update(testDict[stest]) 605 # nsize is special because it is usually overwritten 606 if 'nsize' in testDict[stest]: 607 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 608 else: 609 fh.write("nsize=1\n") 610 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 611 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 612 #if '10_9' in testname: print sLoopVars 613 if sLoopVars: 614 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 615 fh.write(sLoopHead+"\n") 616 fh.write(self.getCmds(subst,j)+"\n") 617 if sLoopVars: 618 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 619 fh.write(sLoopFoot+"\n") 620 else: 621 fh.write(self.getCmds(subst,j)+"\n") 622 623 if loopVars: 624 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 625 fh.write(loopFoot+"\n") 626 627 fh.write(footer+"\n") 628 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 629 #if '10_9' in testname: sys.exit() 630 return 631 632 def genScriptsAndInfo(self,exfile,root,srcDict): 633 """ 634 Generate scripts from the source file, determine if built, etc. 635 For every test in the exfile with info in the srcDict: 636 1. Determine if it needs to be run for this arch 637 2. Generate the script 638 3. Generate the data needed to write out the makefile in a 639 convenient way 640 All tests are *always* run, but some may be SKIP'd per the TAP standard 641 """ 642 debug=False 643 execname=self.getExecname(exfile,root) 644 isBuilt=self._isBuilt(exfile,srcDict) 645 for test in srcDict: 646 if test in self.buildkeys: continue 647 if debug: print self.nameSpace(exfile,root), test 648 srcDict[test]['execname']=execname # Convenience in generating scripts 649 isRun=self._isRun(srcDict[test]) 650 self.genRunScript(test,root,isRun,srcDict) 651 srcDict[test]['isrun']=isRun 652 self.addToTests(test,root,exfile,execname,srcDict[test]) 653 654 # This adds to datastructure for building deps 655 if isBuilt: self.addToSources(exfile,root,srcDict) 656 return 657 658 def _isBuilt(self,exfile,srcDict): 659 """ 660 Determine if this file should be built. 661 """ 662 # Get the language based on file extension 663 srcDict['SKIP'] = [] 664 lang=self.getLanguage(exfile) 665 if (lang=="F" or lang=="F90"): 666 if not self.have_fortran: 667 srcDict["SKIP"].append("Fortran required for this test") 668 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 669 srcDict["SKIP"].append("Fortran f90freeform required for this test") 670 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 671 srcDict["SKIP"].append("CUDA required for this test") 672 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 673 srcDict["SKIP"].append("C++ required for this test") 674 675 # Deprecated source files 676 if srcDict.get("TODO"): 677 return False 678 679 # isRun can work with srcDict to handle the requires 680 if "requires" in srcDict: 681 if srcDict["requires"]: 682 return self._isRun(srcDict) 683 684 return srcDict['SKIP'] == [] 685 686 687 def _isRun(self,testDict, debug=False): 688 """ 689 Based on the requirements listed in the src file and the petscconf.h 690 info, determine whether this test should be run or not. 691 """ 692 indent=" " 693 694 if 'SKIP' not in testDict: 695 testDict['SKIP'] = [] 696 # MPI requirements 697 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 698 if debug: print indent+"Cannot run parallel tests" 699 testDict['SKIP'].append("Parallel test with serial build") 700 701 # The requirements for the test are the sum of all the run subtests 702 if 'subtests' in testDict: 703 if 'requires' not in testDict: testDict['requires']="" 704 for stest in testDict['subtests']: 705 if 'requires' in testDict[stest]: 706 testDict['requires']+=" "+testDict[stest]['requires'] 707 if 'nsize' in testDict[stest]: 708 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 709 testDict['SKIP'].append("Parallel test with serial build") 710 711 712 # Now go through all requirements 713 if 'requires' in testDict: 714 for requirement in testDict['requires'].split(): 715 requirement=requirement.strip() 716 if not requirement: continue 717 if debug: print indent+"Requirement: ", requirement 718 isNull=False 719 if requirement.startswith("!"): 720 requirement=requirement[1:]; isNull=True 721 # Precision requirement for reals 722 if requirement in self.precision_types: 723 if self.conf['PETSC_PRECISION']==requirement: 724 if isNull: 725 testDict['SKIP'].append("not "+requirement+" required") 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append(requirement+" required") 730 continue 731 # Precision requirement for ints 732 if requirement in self.integer_types: 733 if requirement=="int32": 734 if self.conf['PETSC_SIZEOF_INT']==4: 735 if isNull: 736 testDict['SKIP'].append("not int32 required") 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("int32 required") 741 continue 742 if requirement=="int64": 743 if self.conf['PETSC_SIZEOF_INT']==8: 744 if isNull: 745 testDict['SKIP'].append("NOT int64 required") 746 continue 747 continue # Success 748 elif not isNull: 749 testDict['SKIP'].append("int64 required") 750 continue 751 # Datafilespath 752 if requirement=="datafilespath" and not isNull: 753 testDict['SKIP'].append("Requires DATAFILESPATH") 754 continue 755 # Defines -- not sure I have comments matching 756 if "define(" in requirement.lower(): 757 reqdef=requirement.split("(")[1].split(")")[0] 758 if reqdef in self.conf: 759 if isNull: 760 testDict['SKIP'].append("Null requirement not met: "+requirement) 761 continue 762 continue # Success 763 elif not isNull: 764 testDict['SKIP'].append("Required: "+requirement) 765 continue 766 767 # Rest should be packages that we can just get from conf 768 if requirement == "complex": 769 petscconfvar="PETSC_USE_COMPLEX" 770 else: 771 petscconfvar="PETSC_HAVE_"+requirement.upper() 772 if self.conf.get(petscconfvar): 773 if isNull: 774 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 775 continue 776 continue # Success 777 elif not isNull: 778 if debug: print "requirement not found: ", requirement 779 testDict['SKIP'].append(petscconfvar+" requirement not met") 780 continue 781 782 return testDict['SKIP'] == [] 783 784 def genPetscTests_summarize(self,dataDict): 785 """ 786 Required method to state what happened 787 """ 788 if not self.summarize: return 789 indent=" " 790 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 791 fh=open(fhname,"w") 792 #print "See ", fhname 793 for root in dataDict: 794 relroot=self.srcrelpath(root) 795 pkg=relroot.split("/")[1] 796 fh.write(relroot+"\n") 797 allSrcs=[] 798 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 799 for exfile in dataDict[root]: 800 # Basic information 801 rfile=os.path.join(relroot,exfile) 802 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 803 fh.write(indent+exfile+indent*4+builtStatus+"\n") 804 805 for test in dataDict[root][exfile]: 806 if test in self.buildkeys: continue 807 line=indent*2+test 808 fh.write(line+"\n") 809 # Looks nice to have the keys in order 810 #for key in dataDict[root][exfile][test]: 811 for key in "isrun abstracted nsize args requires script".split(): 812 if key not in dataDict[root][exfile][test]: continue 813 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 814 fh.write(line+"\n") 815 fh.write("\n") 816 fh.write("\n") 817 fh.write("\n") 818 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 819 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 820 fh.close() 821 return 822 823 def genPetscTests(self,root,dirs,files,dataDict): 824 """ 825 Go through and parse the source files in the directory to generate 826 the examples based on the metadata contained in the source files 827 """ 828 debug=False 829 # Use examplesAnalyze to get what the makefles think are sources 830 #self.examplesAnalyze(root,dirs,files,anlzDict) 831 832 dataDict[root]={} 833 834 for exfile in files: 835 #TST: Until we replace files, still leaving the orginals as is 836 #if not exfile.startswith("new_"+"ex"): continue 837 #if not exfile.startswith("ex"): continue 838 839 # Ignore emacs and other temporary files 840 if exfile.startswith("."): continue 841 if exfile.startswith("#"): continue 842 843 # Convenience 844 fullex=os.path.join(root,exfile) 845 if self.verbose: print(' --> '+fullex) 846 dataDict[root].update(testparse.parseTestFile(fullex,0)) 847 if exfile in dataDict[root]: 848 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 849 850 return 851 852 def walktree(self,top): 853 """ 854 Walk a directory tree, starting from 'top' 855 """ 856 #print "action", action 857 # Goal of action is to fill this dictionary 858 dataDict={} 859 for root, dirs, files in os.walk(top, topdown=True): 860 if not "examples" in root: continue 861 if "dSYM" in root: continue 862 if os.path.basename(root.rstrip("/")) == 'output': continue 863 if self.verbose: print(root) 864 self.genPetscTests(root,dirs,files,dataDict) 865 # Now summarize this dictionary 866 self.genPetscTests_summarize(dataDict) 867 return dataDict 868 869 def gen_gnumake(self, fd): 870 """ 871 Overwrite of the method in the base PETSc class 872 """ 873 def write(stem, srcs): 874 for lang in LANGS: 875 if srcs[lang]['srcs']: 876 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 877 for pkg in PKGS: 878 srcs = self.gen_pkg(pkg) 879 write('testsrcs-' + pkg, srcs) 880 # Handle dependencies 881 for lang in LANGS: 882 for exfile in srcs[lang]['srcs']: 883 if exfile in srcs[lang]: 884 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 885 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 886 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 887 if deps: 888 # The executable literally depends on the object file because it is linked 889 fd.write(ex +": " + " ".join(deps) +'\n') 890 # The object file containing 'main' does not normally depend on other object 891 # files, but it does when it includes their modules. This dependency is 892 # overly blunt and could be reduced to only depend on object files for 893 # modules that are used, like "*f90aux.o". 894 fd.write(exfo +": " + " ".join(deps) +'\n') 895 896 return self.gendeps 897 898 def gen_pkg(self, pkg): 899 """ 900 Overwrite of the method in the base PETSc class 901 """ 902 return self.sources[pkg] 903 904 def write_gnumake(self,dataDict): 905 """ 906 Write out something similar to files from gmakegen.py 907 908 Test depends on script which also depends on source 909 file, but since I don't have a good way generating 910 acting on a single file (oops) just depend on 911 executable which in turn will depend on src file 912 """ 913 # Different options for how to set up the targets 914 compileExecsFirst=False 915 916 # Open file 917 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 918 fd = open(arch_files, 'w') 919 920 # Write out the sources 921 gendeps = self.gen_gnumake(fd) 922 923 # Write out the tests and execname targets 924 fd.write("\n#Tests and executables\n") # Delimiter 925 926 for pkg in PKGS: 927 # These grab the ones that are built 928 for lang in LANGS: 929 testdeps=[] 930 for ftest in self.tests[pkg][lang]: 931 test=os.path.basename(ftest) 932 basedir=os.path.dirname(ftest) 933 testdeps.append(self.nameSpace(test,basedir)) 934 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 935 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 936 937 # test targets 938 for ftest in self.tests[pkg][lang]: 939 test=os.path.basename(ftest) 940 basedir=os.path.dirname(ftest) 941 testdir="${TESTDIR}/"+basedir+"/" 942 nmtest=self.nameSpace(test,basedir) 943 rundir=os.path.join(testdir,test) 944 #print test, nmtest 945 script=test+".sh" 946 947 # Deps 948 exfile=self.tests[pkg][lang][ftest]['exfile'] 949 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 950 localexec=self.tests[pkg][lang][ftest]['exec'] 951 execname=os.path.join(testdir,localexec) 952 fullscript=os.path.join(testdir,script) 953 tmpfile=os.path.join(testdir,test,test+".tmp") 954 955 # *.counts depends on the script and either executable (will 956 # be run) or the example source file (SKIP or TODO) 957 fd.write('%s.counts : %s %s' 958 % (os.path.join('$(TESTDIR)/counts', nmtest), 959 fullscript, 960 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 961 ) 962 if exfile in self.sources[pkg][lang]: 963 for dep in self.sources[pkg][lang][exfile]: 964 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 965 fd.write('\n') 966 967 # Now write the args: 968 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 969 970 fd.close() 971 return 972 973 def writeHarness(self,output,dataDict): 974 """ 975 This is set up to write out multiple harness even if only gnumake 976 is supported now 977 """ 978 eval("self.write_"+output+"(dataDict)") 979 return 980 981def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 982 if output is None: 983 output = 'gnumake' 984 985 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 986 if petsc_arch: 987 if len(petsc_arch.split(os.path.sep))>1: 988 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 989 990 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 991 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 992 testdir=testdir) 993 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 994 pEx.writeHarness(output,dataDict) 995 996if __name__ == '__main__': 997 import optparse 998 parser = optparse.OptionParser() 999 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1000 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1001 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1002 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1003 parser.add_option('--output', help='Location to write output file', default=None) 1004 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1005 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 1006 1007 opts, extra_args = parser.parse_args() 1008 if extra_args: 1009 import sys 1010 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1011 exit(1) 1012 1013 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1014 output=opts.output, verbose=opts.verbose, 1015 single_ex=opts.single_executable, srcdir=opts.srcdir, 1016 testdir=opts.testdir) 1017