1#!/usr/bin/env python 2 3import os,shutil, string, re 4from distutils.sysconfig import parse_makefile 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 3 modes of running tests: Normal builds, test installs from builds, 23test installs from within install dir. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. Test immediately after build through makefile & lib/petsc/conf/test: 46 (Not in installDir, but using prefix dir. PETSC_ARCH='') 47 48 +---------------------+----------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+----------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+----------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+----------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_SRC_DIR/src | 56 +---------------------+----------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+----------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 60 +---------------------+----------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 62 +---------------------+----------------------------------+ 63 64Case 3. From install dir: 65 66 +---------------------+-------------------------------------------------------+ 67 | PETSC_DIR | <prefix dir> | 68 +---------------------+-------------------------------------------------------+ 69 | PETSC_ARCH | '' | 70 +---------------------+-------------------------------------------------------+ 71 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 72 +---------------------+-------------------------------------------------------+ 73 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 74 +---------------------+-------------------------------------------------------+ 75 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 76 +---------------------+-------------------------------------------------------+ 77 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 78 +---------------------+-------------------------------------------------------+ 79 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 80 +---------------------+-------------------------------------------------------+ 81 82""" 83class generateExamples(Petsc): 84 """ 85 gmakegen.py has basic structure for finding the files, writing out 86 the dependencies, etc. 87 """ 88 def __init__(self,petsc_dir=None, petsc_arch=None, testdir=None, verbose=False, single_ex=False, srcdir=None): 89 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 90 91 self.single_ex=single_ex 92 93 # Set locations to handle movement 94 self.inInstallDir=self.getInInstallDir(thisscriptdir) 95 96 if not self.inInstallDir: 97 if not petsc_arch == '': 98 # Case 1 discussed above 99 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 100 self.srcdir=os.path.join(self.petsc_dir,'src') 101 else: 102 # Case 2 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 105 else: 106 # Case 3 discussed above 107 # set PETSC_ARCH to install directory to allow script to work in both 108 dirlist=thisscriptdir.split(os.path.sep) 109 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 110 self.arch_dir=installdir 111 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 112 113 # Do some initialization 114 if testdir: 115 # If full path given, then use it, otherwise assume relative to arch_dir 116 if testdir.strip().startswith(os.path.sep): 117 self.testroot_dir=testdir.strip() 118 else: 119 self.testroot_dir=os.path.join(self.arch_dir,testdir.strip()) 120 else: 121 self.testroot_dir=os.path.join(self.arch_dir,"tests") 122 123 self.ptNaming=True 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64".split() 131 self.languages="fortran cuda cxx".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in PKGS: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.join('src',os.path.relpath(rdir,self.srcdir)) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def nameSpace(self,srcfile,srcdir): 181 """ 182 Because the scripts have a non-unique naming, the pretty-printing 183 needs to convey the srcdir and srcfile. There are two ways of doing this. 184 """ 185 if self.ptNaming: 186 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 187 cdir=srcdir.split('src')[1].lstrip("/").rstrip("/") 188 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 189 nameString=prefix+srcfile 190 else: 191 #nameString=srcdir+": "+srcfile 192 nameString=srcfile 193 return nameString 194 195 def getLanguage(self,srcfile): 196 """ 197 Based on the source, determine associated language as found in gmakegen.LANGS 198 Can we just return srcext[1:\] now? 199 """ 200 langReq=None 201 srcext=os.path.splitext(srcfile)[-1] 202 if srcext in ".F90".split(): langReq="F90" 203 if srcext in ".F".split(): langReq="F" 204 if srcext in ".cxx".split(): langReq="cxx" 205 if srcext == ".cu": langReq="cu" 206 if srcext == ".c": langReq="c" 207 #if not langReq: print "ERROR: ", srcext, srcfile 208 return langReq 209 210 def _getLoopVars(self,inDict,testname, isSubtest=False): 211 """ 212 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 213 Return: 214 inDict['args']: -ksp_monitor 215 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 216 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 217 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 218 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 219 subst should be passed in instead of inDict 220 """ 221 loopVars={}; newargs="" 222 lkeys=inDict.keys() 223 lsuffix='_' 224 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 225 from testparse import parseLoopArgs 226 for key in lkeys: 227 if type(inDict[key])!=types.StringType: continue 228 keystr = str(inDict[key]) 229 akey=('subargs' if key=='args' else key) # what to assign 230 if akey not in inDict: inDict[akey]='' 231 varlist=[] 232 for varset in argregex.split(keystr): 233 if not varset.strip(): continue 234 if '{{' in varset: 235 keyvar,lvars,ftype=parseLoopArgs(varset) 236 if akey not in loopVars: loopVars[akey]={} 237 varlist.append(keyvar) 238 loopVars[akey][keyvar]=[keyvar,lvars] 239 if akey=='nsize': 240 inDict[akey] = '${' + keyvar + '}' 241 lsuffix+=akey+'-'+inDict[akey]+'_' 242 else: 243 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 244 lsuffix+=keyvar+'-${' + keyvar + '}_' 245 else: 246 if key=='args': newargs+=" -"+varset.strip() 247 if varlist: loopVars[akey]['varlist']=varlist 248 249 250 # For subtests, args are always substituted in (not top level) 251 if isSubtest: 252 inDict['subargs']+=" "+newargs.strip() 253 inDict['args']='' 254 if 'label_suffix' in inDict: 255 inDict['label_suffix']+=lsuffix.rstrip('_') 256 else: 257 inDict['label_suffix']=lsuffix.rstrip('_') 258 else: 259 if loopVars.keys(): 260 inDict['args']=newargs.strip() 261 inDict['label_suffix']=lsuffix.rstrip('_') 262 if loopVars.keys(): 263 return loopVars 264 else: 265 return None 266 267 def getArgLabel(self,testDict): 268 """ 269 In all of the arguments in the test dictionary, create a simple 270 string for searching within the makefile system. For simplicity in 271 search, remove "-", for strings, etc. 272 Also, concatenate the arg commands 273 For now, ignore nsize -- seems hard to search for anyway 274 """ 275 # Collect all of the args associated with a test 276 argStr=("" if 'args' not in testDict else testDict['args']) 277 if 'subtests' in testDict: 278 for stest in testDict["subtests"]: 279 sd=testDict[stest] 280 argStr=argStr+("" if 'args' not in sd else sd['args']) 281 282 # Now go through and cleanup 283 argStr=re.sub('{{(.*?)}}',"",argStr) 284 argStr=re.sub('-'," ",argStr) 285 for digit in string.digits: argStr=re.sub(digit," ",argStr) 286 argStr=re.sub("\.","",argStr) 287 argStr=re.sub(",","",argStr) 288 argStr=re.sub('\+',' ',argStr) 289 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 290 return argStr.strip() 291 292 def addToSources(self,exfile,root,srcDict): 293 """ 294 Put into data structure that allows easy generation of makefile 295 """ 296 rpath=self.srcrelpath(root) 297 pkg=rpath.split(os.path.sep)[1] 298 relpfile=os.path.join(rpath,exfile) 299 lang=self.getLanguage(exfile) 300 if not lang: return 301 self.sources[pkg][lang]['srcs'].append(relpfile) 302 self.sources[pkg][lang][relpfile] = [] 303 if 'depends' in srcDict: 304 depSrcList=srcDict['depends'].split() 305 for depSrc in depSrcList: 306 depObj=os.path.splitext(depSrc)[0]+".o" 307 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 308 309 # In gmakefile, ${TESTDIR} var specifies the object compilation 310 testsdir=self.srcrelpath(root)+"/" 311 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 312 self.objects[pkg].append(objfile) 313 return 314 315 def addToTests(self,test,root,exfile,execname,testDict): 316 """ 317 Put into data structure that allows easy generation of makefile 318 Organized by languages to allow testing of languages 319 """ 320 rpath=self.srcrelpath(root) 321 pkg=rpath.split("/")[1] 322 #nmtest=self.nameSpace(test,root) 323 nmtest=os.path.join(rpath,test) 324 lang=self.getLanguage(exfile) 325 if not lang: return 326 self.tests[pkg][lang][nmtest]={} 327 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 328 self.tests[pkg][lang][nmtest]['exec']=execname 329 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 330 return 331 332 def getExecname(self,exfile,root): 333 """ 334 Generate bash script using template found next to this file. 335 This file is read in at constructor time to avoid file I/O 336 """ 337 rpath=self.srcrelpath(root) 338 if self.single_ex: 339 execname=rpath.split("/")[1]+"-ex" 340 else: 341 execname=os.path.splitext(exfile)[0] 342 return execname 343 344 def getSubstVars(self,testDict,rpath,testname): 345 """ 346 Create a dictionary with all of the variables that get substituted 347 into the template commands found in example_template.py 348 """ 349 subst={} 350 351 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 352 if 'nsize' not in testDict: testDict['nsize']=1 353 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 354 for ak in testparse.acceptedkeys: 355 if ak=='test': continue 356 subst[ak]=(testDict[ak] if ak in testDict else '') 357 358 # Now do other variables 359 subst['execname']=testDict['execname'] 360 if 'filter' in testDict: 361 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 362 363 # Others 364 subst['subargs']='' # Default. For variables override 365 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir),rpath) 366 subst['label_suffix']='' 367 subst['comments']="\n#".join(subst['comments'].split("\n")) 368 if subst['comments']: subst['comments']="#"+subst['comments'] 369 subst['exec']="../"+subst['execname'] 370 subst['testroot']=self.testroot_dir 371 subst['testname']=testname 372 dp = self.conf.get('DATAFILESPATH','') 373 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 374 375 # This is used to label some matrices 376 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 377 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 378 379 # These can have for loops and are treated separately later 380 subst['nsize']=str(subst['nsize']) 381 382 #Conf vars 383 if self.petsc_arch.find('valgrind')>=0: 384 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 385 else: 386 subst['mpiexec']=self.conf['MPIEXEC'] 387 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 388 subst['petsc_arch']=self.petsc_arch 389 if not self.inInstallDir: 390 if not self.petsc_arch == '': 391 # Case 1 392 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 393 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 394 else: 395 # Case 2 396 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 397 subst['PETSC_BINDIR']=os.path.join(os.path.dirname(self.srcdir),'lib','petsc','bin') 398 else: 399 # Case 3 400 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 401 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,self.petsc_arch,'bin') 402 subst['diff']=self.conf['DIFF'] 403 subst['rm']=self.conf['RM'] 404 subst['grep']=self.conf['GREP'] 405 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 406 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 407 408 # Output file is special because of subtests override 409 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 410 if not "_" in defroot: defroot=defroot+"_1" 411 subst['defroot']=defroot 412 subst['label']=self.nameSpace(defroot,subst['srcdir']) 413 subst['redirect_file']=defroot+".tmp" 414 if 'output_file' not in testDict: 415 subst['output_file']="output/"+defroot+".out" 416 # Add in the full path here. 417 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 418 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 419 if not subst['TODO']: 420 print "Warning: "+subst['output_file']+" not found." 421 # Worry about alt files here -- see 422 # src/snes/examples/tutorials/output/ex22*.out 423 altlist=[subst['output_file']] 424 basefile,ext = os.path.splitext(subst['output_file']) 425 for i in range(1,9): 426 altroot=basefile+"_alt" 427 if i > 1: altroot=altroot+"_"+str(i) 428 af=altroot+".out" 429 srcaf=os.path.join(subst['srcdir'],af) 430 fullaf=os.path.join(self.petsc_dir,srcaf) 431 if os.path.isfile(fullaf): altlist.append(srcaf) 432 if len(altlist)>1: subst['altfiles']=altlist 433 #if len(altlist)>1: print "Found alt files: ",altlist 434 435 return subst 436 437 def getCmds(self,subst,i): 438 """ 439 Generate bash script using template found next to this file. 440 This file is read in at constructor time to avoid file I/O 441 """ 442 nindnt=i # the start and has to be consistent with below 443 cmdindnt=self.indent*nindnt 444 cmdLines="" 445 446 # MPI is the default -- but we have a few odd commands 447 if not subst['command']: 448 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 449 else: 450 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 451 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 452 453 cmdLines+=cmdindnt+'if test $res = 0; then\n' 454 diffindnt=self.indent*(nindnt+1) 455 if not subst['filter_output']: 456 if 'altfiles' not in subst: 457 cmd=diffindnt+self._substVars(subst,example_template.difftest) 458 else: 459 # Have to do it by hand a bit because of variable number of alt files 460 rf=subst['redirect_file'] 461 cmd=diffindnt+example_template.difftest.split('@')[0] 462 for i in range(len(subst['altfiles'])): 463 af=subst['altfiles'][i] 464 cmd+=af+' '+rf 465 if i!=len(subst['altfiles'])-1: 466 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 467 cmd+=' || ${diff_exe} ' 468 else: 469 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 470 cmd+=subst['label_suffix']+' ""' # Quotes are painful 471 else: 472 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 473 cmdLines+=cmd+"\n" 474 cmdLines+=cmdindnt+'else\n' 475 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 476 cmdLines+=cmdindnt+'fi\n' 477 return cmdLines 478 479 def _substVars(self,subst,origStr): 480 """ 481 Substitute variables 482 """ 483 Str=origStr 484 for subkey in subst: 485 if type(subst[subkey])!=types.StringType: continue 486 patt="@"+subkey.upper()+"@" 487 Str=re.sub(patt,subst[subkey],Str) 488 return Str 489 490 def _writeTodoSkip(self,fh,tors,reasons,footer): 491 """ 492 Write out the TODO and SKIP lines in the file 493 The TODO or SKIP variable, tors, should be lower case 494 """ 495 TORS=tors.upper() 496 template=eval("example_template."+tors+"line") 497 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 498 tab = '' 499 if reasons: 500 fh.write('if ! $force; then\n') 501 tab = tab + ' ' 502 if reasons == ["Requires DATAFILESPATH"]: 503 # The only reason not to run is DATAFILESPATH, which we check at run-time 504 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 505 tab = tab + ' ' 506 if reasons: 507 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 508 fh.write(tab+footer+"\n") 509 fh.write(tab+"exit\n") 510 if reasons == ["Requires DATAFILESPATH"]: 511 fh.write(' fi\n') 512 if reasons: 513 fh.write('fi\n') 514 fh.write('\n\n') 515 return 516 517 def getLoopVarsHead(self,loopVars,i): 518 """ 519 Generate a nicely indented string with the format loops 520 Here is what the data structure looks like 521 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 522 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 523 loopVars['subargs']['pc_type']=["j","cholesky sor"] 524 """ 525 outstr=''; indnt=self.indent 526 for key in loopVars: 527 for var in loopVars[key]['varlist']: 528 varval=loopVars[key][var] 529 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 530 i = i + 1 531 return (outstr,i) 532 533 def getLoopVarsFoot(self,loopVars,i): 534 outstr=''; indnt=self.indent 535 for key in loopVars: 536 for var in loopVars[key]['varlist']: 537 i = i - 1 538 outstr += indnt * i + "done\n" 539 return (outstr,i) 540 541 def genRunScript(self,testname,root,isRun,srcDict): 542 """ 543 Generate bash script using template found next to this file. 544 This file is read in at constructor time to avoid file I/O 545 """ 546 # runscript_dir directory has to be consistent with gmakefile 547 testDict=srcDict[testname] 548 rpath=self.srcrelpath(root) 549 runscript_dir=os.path.join(self.testroot_dir,rpath) 550 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 551 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 552 553 # Get variables to go into shell scripts. last time testDict used 554 subst=self.getSubstVars(testDict,rpath,testname) 555 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 556 #if '33_' in testname: print subst['subargs'] 557 558 #Handle runfiles 559 for lfile in subst.get('localrunfiles','').split(): 560 fullfile=os.path.join(root,lfile) 561 if os.path.isdir(fullfile): 562 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 563 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 564 else: 565 shutil.copy(fullfile,runscript_dir) 566 # Check subtests for local runfiles 567 for stest in subst.get("subtests",[]): 568 for lfile in testDict[stest].get('localrunfiles','').split(): 569 fullfile=os.path.join(root,lfile) 570 if os.path.isdir(fullfile): 571 if not os.path.isdir(os.path.join(runscript_dir,lfile)): 572 shutil.copytree(fullfile,os.path.join(runscript_dir,lfile)) 573 else: 574 shutil.copy(fullfile,self.runscript_dir) 575 576 # Now substitute the key variables into the header and footer 577 header=self._substVars(subst,example_template.header) 578 # The header is done twice to enable @...@ in header 579 header=self._substVars(subst,header) 580 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 581 582 # Start writing the file 583 fh.write(header+"\n") 584 585 # If there is a TODO or a SKIP then we do it before writing out the 586 # rest of the command (which is useful for working on the test) 587 # SKIP and TODO can be for the source file or for the runs 588 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 589 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 590 591 j=0 # for indentation 592 593 if loopVars: 594 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 595 if (loopHead): fh.write(loopHead+"\n") 596 597 # Subtests are special 598 if 'subtests' in testDict: 599 substP=subst # Subtests can inherit args but be careful 600 k=0 # for label suffixes 601 for stest in testDict["subtests"]: 602 subst=substP.copy() 603 subst.update(testDict[stest]) 604 # nsize is special because it is usually overwritten 605 if 'nsize' in testDict[stest]: 606 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 607 else: 608 fh.write("nsize=1\n") 609 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 610 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 611 #if '10_9' in testname: print sLoopVars 612 if sLoopVars: 613 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 614 fh.write(sLoopHead+"\n") 615 fh.write(self.getCmds(subst,j)+"\n") 616 if sLoopVars: 617 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 618 fh.write(sLoopFoot+"\n") 619 else: 620 fh.write(self.getCmds(subst,j)+"\n") 621 622 if loopVars: 623 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 624 fh.write(loopFoot+"\n") 625 626 fh.write(footer+"\n") 627 os.chmod(os.path.join(runscript_dir,testname+".sh"),0755) 628 #if '10_9' in testname: sys.exit() 629 return 630 631 def genScriptsAndInfo(self,exfile,root,srcDict): 632 """ 633 Generate scripts from the source file, determine if built, etc. 634 For every test in the exfile with info in the srcDict: 635 1. Determine if it needs to be run for this arch 636 2. Generate the script 637 3. Generate the data needed to write out the makefile in a 638 convenient way 639 All tests are *always* run, but some may be SKIP'd per the TAP standard 640 """ 641 debug=False 642 execname=self.getExecname(exfile,root) 643 isBuilt=self._isBuilt(exfile,srcDict) 644 for test in srcDict: 645 if test in self.buildkeys: continue 646 if debug: print self.nameSpace(exfile,root), test 647 srcDict[test]['execname']=execname # Convenience in generating scripts 648 isRun=self._isRun(srcDict[test]) 649 self.genRunScript(test,root,isRun,srcDict) 650 srcDict[test]['isrun']=isRun 651 self.addToTests(test,root,exfile,execname,srcDict[test]) 652 653 # This adds to datastructure for building deps 654 if isBuilt: self.addToSources(exfile,root,srcDict) 655 return 656 657 def _isBuilt(self,exfile,srcDict): 658 """ 659 Determine if this file should be built. 660 """ 661 # Get the language based on file extension 662 srcDict['SKIP'] = [] 663 lang=self.getLanguage(exfile) 664 if (lang=="F" or lang=="F90"): 665 if not self.have_fortran: 666 srcDict["SKIP"].append("Fortran required for this test") 667 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 668 srcDict["SKIP"].append("Fortran f90freeform required for this test") 669 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 670 srcDict["SKIP"].append("CUDA required for this test") 671 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 672 srcDict["SKIP"].append("C++ required for this test") 673 674 # Deprecated source files 675 if srcDict.get("TODO"): 676 return False 677 678 # isRun can work with srcDict to handle the requires 679 if "requires" in srcDict: 680 if srcDict["requires"]: 681 return self._isRun(srcDict) 682 683 return srcDict['SKIP'] == [] 684 685 686 def _isRun(self,testDict, debug=False): 687 """ 688 Based on the requirements listed in the src file and the petscconf.h 689 info, determine whether this test should be run or not. 690 """ 691 indent=" " 692 693 if 'SKIP' not in testDict: 694 testDict['SKIP'] = [] 695 # MPI requirements 696 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 697 if debug: print indent+"Cannot run parallel tests" 698 testDict['SKIP'].append("Parallel test with serial build") 699 700 # The requirements for the test are the sum of all the run subtests 701 if 'subtests' in testDict: 702 if 'requires' not in testDict: testDict['requires']="" 703 for stest in testDict['subtests']: 704 if 'requires' in testDict[stest]: 705 testDict['requires']+=" "+testDict[stest]['requires'] 706 if 'nsize' in testDict[stest]: 707 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 708 testDict['SKIP'].append("Parallel test with serial build") 709 710 711 # Now go through all requirements 712 if 'requires' in testDict: 713 for requirement in testDict['requires'].split(): 714 requirement=requirement.strip() 715 if not requirement: continue 716 if debug: print indent+"Requirement: ", requirement 717 isNull=False 718 if requirement.startswith("!"): 719 requirement=requirement[1:]; isNull=True 720 # Precision requirement for reals 721 if requirement in self.precision_types: 722 if self.conf['PETSC_PRECISION']==requirement: 723 if isNull: 724 testDict['SKIP'].append("not "+requirement+" required") 725 continue 726 continue # Success 727 elif not isNull: 728 testDict['SKIP'].append(requirement+" required") 729 continue 730 # Precision requirement for ints 731 if requirement in self.integer_types: 732 if requirement=="int32": 733 if self.conf['PETSC_SIZEOF_INT']==4: 734 if isNull: 735 testDict['SKIP'].append("not int32 required") 736 continue 737 continue # Success 738 elif not isNull: 739 testDict['SKIP'].append("int32 required") 740 continue 741 if requirement=="int64": 742 if self.conf['PETSC_SIZEOF_INT']==8: 743 if isNull: 744 testDict['SKIP'].append("NOT int64 required") 745 continue 746 continue # Success 747 elif not isNull: 748 testDict['SKIP'].append("int64 required") 749 continue 750 # Datafilespath 751 if requirement=="datafilespath" and not isNull: 752 testDict['SKIP'].append("Requires DATAFILESPATH") 753 continue 754 # Defines -- not sure I have comments matching 755 if "define(" in requirement.lower(): 756 reqdef=requirement.split("(")[1].split(")")[0] 757 if reqdef in self.conf: 758 if isNull: 759 testDict['SKIP'].append("Null requirement not met: "+requirement) 760 continue 761 continue # Success 762 elif not isNull: 763 testDict['SKIP'].append("Required: "+requirement) 764 continue 765 766 # Rest should be packages that we can just get from conf 767 if requirement == "complex": 768 petscconfvar="PETSC_USE_COMPLEX" 769 else: 770 petscconfvar="PETSC_HAVE_"+requirement.upper() 771 if self.conf.get(petscconfvar): 772 if isNull: 773 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 774 continue 775 continue # Success 776 elif not isNull: 777 if debug: print "requirement not found: ", requirement 778 testDict['SKIP'].append(petscconfvar+" requirement not met") 779 continue 780 781 return testDict['SKIP'] == [] 782 783 def genPetscTests_summarize(self,dataDict): 784 """ 785 Required method to state what happened 786 """ 787 if not self.summarize: return 788 indent=" " 789 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 790 fh=open(fhname,"w") 791 #print "See ", fhname 792 for root in dataDict: 793 relroot=self.srcrelpath(root) 794 pkg=relroot.split("/")[1] 795 fh.write(relroot+"\n") 796 allSrcs=[] 797 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 798 for exfile in dataDict[root]: 799 # Basic information 800 rfile=os.path.join(relroot,exfile) 801 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 802 fh.write(indent+exfile+indent*4+builtStatus+"\n") 803 804 for test in dataDict[root][exfile]: 805 if test in self.buildkeys: continue 806 line=indent*2+test 807 fh.write(line+"\n") 808 # Looks nice to have the keys in order 809 #for key in dataDict[root][exfile][test]: 810 for key in "isrun abstracted nsize args requires script".split(): 811 if key not in dataDict[root][exfile][test]: continue 812 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 813 fh.write(line+"\n") 814 fh.write("\n") 815 fh.write("\n") 816 fh.write("\n") 817 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 818 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 819 fh.close() 820 return 821 822 def genPetscTests(self,root,dirs,files,dataDict): 823 """ 824 Go through and parse the source files in the directory to generate 825 the examples based on the metadata contained in the source files 826 """ 827 debug=False 828 # Use examplesAnalyze to get what the makefles think are sources 829 #self.examplesAnalyze(root,dirs,files,anlzDict) 830 831 dataDict[root]={} 832 833 for exfile in files: 834 #TST: Until we replace files, still leaving the orginals as is 835 #if not exfile.startswith("new_"+"ex"): continue 836 #if not exfile.startswith("ex"): continue 837 838 # Ignore emacs and other temporary files 839 if exfile.startswith("."): continue 840 if exfile.startswith("#"): continue 841 if exfile.endswith("~"): continue 842 843 # Convenience 844 fullex=os.path.join(root,exfile) 845 if self.verbose: print(' --> '+fullex) 846 dataDict[root].update(testparse.parseTestFile(fullex,0)) 847 if exfile in dataDict[root]: 848 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 849 850 return 851 852 def walktree(self,top): 853 """ 854 Walk a directory tree, starting from 'top' 855 """ 856 #print "action", action 857 # Goal of action is to fill this dictionary 858 dataDict={} 859 for root, dirs, files in os.walk(top, topdown=True): 860 if not "examples" in root: continue 861 if "dSYM" in root: continue 862 if os.path.basename(root.rstrip("/")) == 'output': continue 863 if self.verbose: print(root) 864 self.genPetscTests(root,dirs,files,dataDict) 865 # Now summarize this dictionary 866 self.genPetscTests_summarize(dataDict) 867 return dataDict 868 869 def gen_gnumake(self, fd): 870 """ 871 Overwrite of the method in the base PETSc class 872 """ 873 def write(stem, srcs): 874 for lang in LANGS: 875 if srcs[lang]['srcs']: 876 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 877 for pkg in PKGS: 878 srcs = self.gen_pkg(pkg) 879 write('testsrcs-' + pkg, srcs) 880 # Handle dependencies 881 for lang in LANGS: 882 for exfile in srcs[lang]['srcs']: 883 if exfile in srcs[lang]: 884 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 885 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 886 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 887 if deps: 888 # The executable literally depends on the object file because it is linked 889 fd.write(ex +": " + " ".join(deps) +'\n') 890 # The object file containing 'main' does not normally depend on other object 891 # files, but it does when it includes their modules. This dependency is 892 # overly blunt and could be reduced to only depend on object files for 893 # modules that are used, like "*f90aux.o". 894 fd.write(exfo +": " + " ".join(deps) +'\n') 895 896 return self.gendeps 897 898 def gen_pkg(self, pkg): 899 """ 900 Overwrite of the method in the base PETSc class 901 """ 902 return self.sources[pkg] 903 904 def write_gnumake(self,dataDict): 905 """ 906 Write out something similar to files from gmakegen.py 907 908 Test depends on script which also depends on source 909 file, but since I don't have a good way generating 910 acting on a single file (oops) just depend on 911 executable which in turn will depend on src file 912 """ 913 # Different options for how to set up the targets 914 compileExecsFirst=False 915 916 # Open file 917 arch_files = os.path.join(self.arch_dir,'lib','petsc','conf', 'testfiles') 918 fd = open(arch_files, 'w') 919 920 # Write out the sources 921 gendeps = self.gen_gnumake(fd) 922 923 # Write out the tests and execname targets 924 fd.write("\n#Tests and executables\n") # Delimiter 925 926 for pkg in PKGS: 927 # These grab the ones that are built 928 for lang in LANGS: 929 testdeps=[] 930 for ftest in self.tests[pkg][lang]: 931 test=os.path.basename(ftest) 932 basedir=os.path.dirname(ftest) 933 testdeps.append(self.nameSpace(test,basedir)) 934 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 935 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 936 937 # test targets 938 for ftest in self.tests[pkg][lang]: 939 test=os.path.basename(ftest) 940 basedir=os.path.dirname(ftest) 941 testdir="${TESTDIR}/"+basedir+"/" 942 nmtest=self.nameSpace(test,basedir) 943 rundir=os.path.join(testdir,test) 944 #print test, nmtest 945 script=test+".sh" 946 947 # Deps 948 exfile=self.tests[pkg][lang][ftest]['exfile'] 949 fullex=os.path.join(os.path.dirname(self.srcdir),exfile) 950 localexec=self.tests[pkg][lang][ftest]['exec'] 951 execname=os.path.join(testdir,localexec) 952 fullscript=os.path.join(testdir,script) 953 tmpfile=os.path.join(testdir,test,test+".tmp") 954 955 # *.counts depends on the script and either executable (will 956 # be run) or the example source file (SKIP or TODO) 957 fd.write('%s.counts : %s %s' 958 % (os.path.join('$(TESTDIR)/counts', nmtest), 959 fullscript, 960 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 961 ) 962 if exfile in self.sources[pkg][lang]: 963 for dep in self.sources[pkg][lang][exfile]: 964 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 965 fd.write('\n') 966 967 # Now write the args: 968 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 969 970 fd.close() 971 return 972 973 def writeHarness(self,output,dataDict): 974 """ 975 This is set up to write out multiple harness even if only gnumake 976 is supported now 977 """ 978 eval("self.write_"+output+"(dataDict)") 979 return 980 981def main(petsc_dir=None, petsc_arch=None, output=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 982 if output is None: 983 output = 'gnumake' 984 985 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 986 if petsc_arch: 987 if len(petsc_arch.split(os.path.sep))>1: 988 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 989 990 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 991 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 992 testdir=testdir) 993 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 994 pEx.writeHarness(output,dataDict) 995 996if __name__ == '__main__': 997 import optparse 998 parser = optparse.OptionParser() 999 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1000 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1001 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1002 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1003 parser.add_option('--output', help='Location to write output file', default=None) 1004 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1005 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory: PETSC_DIR/PETSC_ARCH/testdir. Default is "tests"') 1006 1007 opts, extra_args = parser.parse_args() 1008 if extra_args: 1009 import sys 1010 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1011 exit(1) 1012 1013 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1014 output=opts.output, verbose=opts.verbose, 1015 single_ex=opts.single_executable, srcdir=opts.srcdir, 1016 testdir=opts.testdir) 1017