1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print "ERROR: ", srcext, srcfile 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs="" 206 lkeys=inDict.keys() 207 lsuffix='_' 208 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 209 from testparse import parseLoopArgs 210 for key in lkeys: 211 if type(inDict[key])!=bytes: continue 212 keystr = str(inDict[key]) 213 akey=('subargs' if key=='args' else key) # what to assign 214 if akey not in inDict: inDict[akey]='' 215 varlist=[] 216 for varset in argregex.split(keystr): 217 if not varset.strip(): continue 218 if '{{' in varset: 219 keyvar,lvars,ftype=parseLoopArgs(varset) 220 if akey not in loopVars: loopVars[akey]={} 221 varlist.append(keyvar) 222 loopVars[akey][keyvar]=[keyvar,lvars] 223 if akey=='nsize': 224 inDict[akey] = '${' + keyvar + '}' 225 lsuffix+=akey+'-'+inDict[akey]+'_' 226 else: 227 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 228 lsuffix+=keyvar+'-${' + keyvar + '}_' 229 else: 230 if key=='args': newargs+=" -"+varset.strip() 231 if varlist: loopVars[akey]['varlist']=varlist 232 233 234 # For subtests, args are always substituted in (not top level) 235 if isSubtest: 236 inDict['subargs']+=" "+newargs.strip() 237 inDict['args']='' 238 if 'label_suffix' in inDict: 239 inDict['label_suffix']+=lsuffix.rstrip('_') 240 else: 241 inDict['label_suffix']=lsuffix.rstrip('_') 242 else: 243 if loopVars.keys(): 244 inDict['args']=newargs.strip() 245 inDict['label_suffix']=lsuffix.rstrip('_') 246 if loopVars.keys(): 247 return loopVars 248 else: 249 return None 250 251 def getArgLabel(self,testDict): 252 """ 253 In all of the arguments in the test dictionary, create a simple 254 string for searching within the makefile system. For simplicity in 255 search, remove "-", for strings, etc. 256 Also, concatenate the arg commands 257 For now, ignore nsize -- seems hard to search for anyway 258 """ 259 # Collect all of the args associated with a test 260 argStr=("" if 'args' not in testDict else testDict['args']) 261 if 'subtests' in testDict: 262 for stest in testDict["subtests"]: 263 sd=testDict[stest] 264 argStr=argStr+("" if 'args' not in sd else sd['args']) 265 266 # Now go through and cleanup 267 argStr=re.sub('{{(.*?)}}',"",argStr) 268 argStr=re.sub('-'," ",argStr) 269 for digit in string.digits: argStr=re.sub(digit," ",argStr) 270 argStr=re.sub("\.","",argStr) 271 argStr=re.sub(",","",argStr) 272 argStr=re.sub('\+',' ',argStr) 273 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 274 return argStr.strip() 275 276 def addToSources(self,exfile,root,srcDict): 277 """ 278 Put into data structure that allows easy generation of makefile 279 """ 280 rpath=self.srcrelpath(root) 281 pkg=rpath.split(os.path.sep)[0] 282 relpfile=os.path.join(rpath,exfile) 283 lang=self.getLanguage(exfile) 284 if not lang: return 285 self.sources[pkg][lang]['srcs'].append(relpfile) 286 self.sources[pkg][lang][relpfile] = [] 287 if 'depends' in srcDict: 288 depSrcList=srcDict['depends'].split() 289 for depSrc in depSrcList: 290 depObj=os.path.splitext(depSrc)[0]+".o" 291 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 292 293 # In gmakefile, ${TESTDIR} var specifies the object compilation 294 testsdir=self.srcrelpath(root)+"/" 295 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 296 self.objects[pkg].append(objfile) 297 return 298 299 def addToTests(self,test,root,exfile,execname,testDict): 300 """ 301 Put into data structure that allows easy generation of makefile 302 Organized by languages to allow testing of languages 303 """ 304 rpath=self.srcrelpath(root) 305 pkg=rpath.split("/")[0] 306 #nmtest=self.nameSpace(test,root) 307 nmtest=os.path.join(rpath,test) 308 lang=self.getLanguage(exfile) 309 if not lang: return 310 self.tests[pkg][lang][nmtest]={} 311 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 312 self.tests[pkg][lang][nmtest]['exec']=execname 313 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 314 return 315 316 def getExecname(self,exfile,root): 317 """ 318 Generate bash script using template found next to this file. 319 This file is read in at constructor time to avoid file I/O 320 """ 321 rpath=self.srcrelpath(root) 322 if self.single_ex: 323 execname=rpath.split("/")[1]+"-ex" 324 else: 325 execname=os.path.splitext(exfile)[0] 326 return execname 327 328 def getSubstVars(self,testDict,rpath,testname): 329 """ 330 Create a dictionary with all of the variables that get substituted 331 into the template commands found in example_template.py 332 """ 333 subst={} 334 335 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 336 if 'nsize' not in testDict: testDict['nsize']=1 337 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 338 for ak in testparse.acceptedkeys: 339 if ak=='test': continue 340 subst[ak]=(testDict[ak] if ak in testDict else '') 341 342 # Now do other variables 343 subst['execname']=testDict['execname'] 344 if 'filter' in testDict: 345 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 346 347 # Others 348 subst['subargs']='' # Default. For variables override 349 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 350 subst['label_suffix']='' 351 subst['comments']="\n#".join(subst['comments'].split("\n")) 352 if subst['comments']: subst['comments']="#"+subst['comments'] 353 subst['exec']="../"+subst['execname'] 354 subst['testroot']=self.testroot_dir 355 subst['testname']=testname 356 dp = self.conf.get('DATAFILESPATH','') 357 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 358 359 # This is used to label some matrices 360 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 361 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 362 363 # These can have for loops and are treated separately later 364 subst['nsize']=str(subst['nsize']) 365 366 #Conf vars 367 if self.petsc_arch.find('valgrind')>=0: 368 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 369 else: 370 subst['mpiexec']=self.conf['MPIEXEC'] 371 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 372 subst['petsc_arch']=self.petsc_arch 373 if self.inInstallDir: 374 # Case 2 375 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 376 else: 377 # Case 1 378 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 379 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 380 subst['diff']=self.conf['DIFF'] 381 subst['rm']=self.conf['RM'] 382 subst['grep']=self.conf['GREP'] 383 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 384 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 385 386 # Output file is special because of subtests override 387 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 388 if not "_" in defroot: defroot=defroot+"_1" 389 subst['defroot']=defroot 390 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 391 subst['redirect_file']=defroot+".tmp" 392 if 'output_file' not in testDict: 393 subst['output_file']="output/"+defroot+".out" 394 # Add in the full path here. 395 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 396 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 397 if not subst['TODO']: 398 print("Warning: "+subst['output_file']+" not found.") 399 # Worry about alt files here -- see 400 # src/snes/examples/tutorials/output/ex22*.out 401 altlist=[subst['output_file']] 402 basefile,ext = os.path.splitext(subst['output_file']) 403 for i in range(1,9): 404 altroot=basefile+"_alt" 405 if i > 1: altroot=altroot+"_"+str(i) 406 af=altroot+".out" 407 srcaf=os.path.join(subst['srcdir'],af) 408 fullaf=os.path.join(self.petsc_dir,srcaf) 409 if os.path.isfile(fullaf): altlist.append(srcaf) 410 if len(altlist)>1: subst['altfiles']=altlist 411 #if len(altlist)>1: print "Found alt files: ",altlist 412 413 return subst 414 415 def getCmds(self,subst,i): 416 """ 417 Generate bash script using template found next to this file. 418 This file is read in at constructor time to avoid file I/O 419 """ 420 nindnt=i # the start and has to be consistent with below 421 cmdindnt=self.indent*nindnt 422 cmdLines="" 423 424 # MPI is the default -- but we have a few odd commands 425 if not subst['command']: 426 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 427 else: 428 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 429 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 430 431 cmdLines+=cmdindnt+'if test $res = 0; then\n' 432 diffindnt=self.indent*(nindnt+1) 433 if not subst['filter_output']: 434 if 'altfiles' not in subst: 435 cmd=diffindnt+self._substVars(subst,example_template.difftest) 436 else: 437 # Have to do it by hand a bit because of variable number of alt files 438 rf=subst['redirect_file'] 439 cmd=diffindnt+example_template.difftest.split('@')[0] 440 for i in range(len(subst['altfiles'])): 441 af=subst['altfiles'][i] 442 cmd+=af+' '+rf 443 if i!=len(subst['altfiles'])-1: 444 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 445 cmd+=' || ${diff_exe} ' 446 else: 447 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 448 cmd+=subst['label_suffix']+' ""' # Quotes are painful 449 else: 450 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 451 cmdLines+=cmd+"\n" 452 cmdLines+=cmdindnt+'else\n' 453 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 454 cmdLines+=cmdindnt+'fi\n' 455 return cmdLines 456 457 def _substVars(self,subst,origStr): 458 """ 459 Substitute variables 460 """ 461 Str=origStr 462 for subkey in subst: 463 if type(subst[subkey])!=bytes: continue 464 patt="@"+subkey.upper()+"@" 465 Str=re.sub(patt,subst[subkey],Str) 466 return Str 467 468 def _writeTodoSkip(self,fh,tors,reasons,footer): 469 """ 470 Write out the TODO and SKIP lines in the file 471 The TODO or SKIP variable, tors, should be lower case 472 """ 473 TORS=tors.upper() 474 template=eval("example_template."+tors+"line") 475 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 476 tab = '' 477 if reasons: 478 fh.write('if ! $force; then\n') 479 tab = tab + ' ' 480 if reasons == ["Requires DATAFILESPATH"]: 481 # The only reason not to run is DATAFILESPATH, which we check at run-time 482 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 483 tab = tab + ' ' 484 if reasons: 485 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 486 fh.write(tab+footer+"\n") 487 fh.write(tab+"exit\n") 488 if reasons == ["Requires DATAFILESPATH"]: 489 fh.write(' fi\n') 490 if reasons: 491 fh.write('fi\n') 492 fh.write('\n\n') 493 return 494 495 def getLoopVarsHead(self,loopVars,i): 496 """ 497 Generate a nicely indented string with the format loops 498 Here is what the data structure looks like 499 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 500 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 501 loopVars['subargs']['pc_type']=["j","cholesky sor"] 502 """ 503 outstr=''; indnt=self.indent 504 for key in loopVars: 505 for var in loopVars[key]['varlist']: 506 varval=loopVars[key][var] 507 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 508 i = i + 1 509 return (outstr,i) 510 511 def getLoopVarsFoot(self,loopVars,i): 512 outstr=''; indnt=self.indent 513 for key in loopVars: 514 for var in loopVars[key]['varlist']: 515 i = i - 1 516 outstr += indnt * i + "done\n" 517 return (outstr,i) 518 519 def genRunScript(self,testname,root,isRun,srcDict): 520 """ 521 Generate bash script using template found next to this file. 522 This file is read in at constructor time to avoid file I/O 523 """ 524 # runscript_dir directory has to be consistent with gmakefile 525 testDict=srcDict[testname] 526 rpath=self.srcrelpath(root) 527 runscript_dir=os.path.join(self.testroot_dir,rpath) 528 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 529 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 530 531 # Get variables to go into shell scripts. last time testDict used 532 subst=self.getSubstVars(testDict,rpath,testname) 533 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 534 #if '33_' in testname: print subst['subargs'] 535 536 #Handle runfiles 537 for lfile in subst.get('localrunfiles','').split(): 538 install_files(os.path.join(root, lfile), 539 os.path.join(runscript_dir, os.path.dirname(lfile))) 540 # Check subtests for local runfiles 541 for stest in subst.get("subtests",[]): 542 for lfile in testDict[stest].get('localrunfiles','').split(): 543 install_files(os.path.join(root, lfile), 544 os.path.join(runscript_dir, os.path.dirname(lfile))) 545 546 # Now substitute the key variables into the header and footer 547 header=self._substVars(subst,example_template.header) 548 # The header is done twice to enable @...@ in header 549 header=self._substVars(subst,header) 550 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 551 552 # Start writing the file 553 fh.write(header+"\n") 554 555 # If there is a TODO or a SKIP then we do it before writing out the 556 # rest of the command (which is useful for working on the test) 557 # SKIP and TODO can be for the source file or for the runs 558 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 559 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 560 561 j=0 # for indentation 562 563 if loopVars: 564 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 565 if (loopHead): fh.write(loopHead+"\n") 566 567 # Subtests are special 568 if 'subtests' in testDict: 569 substP=subst # Subtests can inherit args but be careful 570 k=0 # for label suffixes 571 for stest in testDict["subtests"]: 572 subst=substP.copy() 573 subst.update(testDict[stest]) 574 # nsize is special because it is usually overwritten 575 if 'nsize' in testDict[stest]: 576 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 577 else: 578 fh.write("nsize=1\n") 579 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 580 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 581 #if '10_9' in testname: print sLoopVars 582 if sLoopVars: 583 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 584 fh.write(sLoopHead+"\n") 585 fh.write(self.getCmds(subst,j)+"\n") 586 if sLoopVars: 587 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 588 fh.write(sLoopFoot+"\n") 589 else: 590 fh.write(self.getCmds(subst,j)+"\n") 591 592 if loopVars: 593 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 594 fh.write(loopFoot+"\n") 595 596 fh.write(footer+"\n") 597 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 598 #if '10_9' in testname: sys.exit() 599 return 600 601 def genScriptsAndInfo(self,exfile,root,srcDict): 602 """ 603 Generate scripts from the source file, determine if built, etc. 604 For every test in the exfile with info in the srcDict: 605 1. Determine if it needs to be run for this arch 606 2. Generate the script 607 3. Generate the data needed to write out the makefile in a 608 convenient way 609 All tests are *always* run, but some may be SKIP'd per the TAP standard 610 """ 611 debug=False 612 execname=self.getExecname(exfile,root) 613 isBuilt=self._isBuilt(exfile,srcDict) 614 for test in srcDict: 615 if test in self.buildkeys: continue 616 if debug: print(self.nameSpace(exfile,root), test) 617 srcDict[test]['execname']=execname # Convenience in generating scripts 618 isRun=self._isRun(srcDict[test]) 619 self.genRunScript(test,root,isRun,srcDict) 620 srcDict[test]['isrun']=isRun 621 self.addToTests(test,root,exfile,execname,srcDict[test]) 622 623 # This adds to datastructure for building deps 624 if isBuilt: self.addToSources(exfile,root,srcDict) 625 return 626 627 def _isBuilt(self,exfile,srcDict): 628 """ 629 Determine if this file should be built. 630 """ 631 # Get the language based on file extension 632 srcDict['SKIP'] = [] 633 lang=self.getLanguage(exfile) 634 if (lang=="F" or lang=="F90"): 635 if not self.have_fortran: 636 srcDict["SKIP"].append("Fortran required for this test") 637 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 638 srcDict["SKIP"].append("Fortran f90freeform required for this test") 639 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 640 srcDict["SKIP"].append("CUDA required for this test") 641 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 642 srcDict["SKIP"].append("C++ required for this test") 643 644 # Deprecated source files 645 if srcDict.get("TODO"): 646 return False 647 648 # isRun can work with srcDict to handle the requires 649 if "requires" in srcDict: 650 if srcDict["requires"]: 651 return self._isRun(srcDict) 652 653 return srcDict['SKIP'] == [] 654 655 656 def _isRun(self,testDict, debug=False): 657 """ 658 Based on the requirements listed in the src file and the petscconf.h 659 info, determine whether this test should be run or not. 660 """ 661 indent=" " 662 663 if 'SKIP' not in testDict: 664 testDict['SKIP'] = [] 665 # MPI requirements 666 if testDict.get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 667 if debug: print(indent+"Cannot run parallel tests") 668 testDict['SKIP'].append("Parallel test with serial build") 669 670 # The requirements for the test are the sum of all the run subtests 671 if 'subtests' in testDict: 672 if 'requires' not in testDict: testDict['requires']="" 673 for stest in testDict['subtests']: 674 if 'requires' in testDict[stest]: 675 testDict['requires']+=" "+testDict[stest]['requires'] 676 if 'nsize' in testDict[stest]: 677 if testDict[stest].get('nsize',1)>1 and 'MPI_IS_MPIUNI' in self.conf: 678 testDict['SKIP'].append("Parallel test with serial build") 679 680 681 # Now go through all requirements 682 if 'requires' in testDict: 683 for requirement in testDict['requires'].split(): 684 requirement=requirement.strip() 685 if not requirement: continue 686 if debug: print(indent+"Requirement: ", requirement) 687 isNull=False 688 if requirement.startswith("!"): 689 requirement=requirement[1:]; isNull=True 690 # Precision requirement for reals 691 if requirement in self.precision_types: 692 if self.conf['PETSC_PRECISION']==requirement: 693 if isNull: 694 testDict['SKIP'].append("not "+requirement+" required") 695 continue 696 continue # Success 697 elif not isNull: 698 testDict['SKIP'].append(requirement+" required") 699 continue 700 # Precision requirement for ints 701 if requirement in self.integer_types: 702 if requirement=="int32": 703 if self.conf['PETSC_SIZEOF_INT']==4: 704 if isNull: 705 testDict['SKIP'].append("not int32 required") 706 continue 707 continue # Success 708 elif not isNull: 709 testDict['SKIP'].append("int32 required") 710 continue 711 if requirement=="int64": 712 if self.conf['PETSC_SIZEOF_INT']==8: 713 if isNull: 714 testDict['SKIP'].append("NOT int64 required") 715 continue 716 continue # Success 717 elif not isNull: 718 testDict['SKIP'].append("int64 required") 719 continue 720 # Datafilespath 721 if requirement=="datafilespath" and not isNull: 722 testDict['SKIP'].append("Requires DATAFILESPATH") 723 continue 724 # Defines -- not sure I have comments matching 725 if "define(" in requirement.lower(): 726 reqdef=requirement.split("(")[1].split(")")[0] 727 if reqdef in self.conf: 728 if isNull: 729 testDict['SKIP'].append("Null requirement not met: "+requirement) 730 continue 731 continue # Success 732 elif not isNull: 733 testDict['SKIP'].append("Required: "+requirement) 734 continue 735 736 # Rest should be packages that we can just get from conf 737 if requirement == "complex": 738 petscconfvar="PETSC_USE_COMPLEX" 739 else: 740 petscconfvar="PETSC_HAVE_"+requirement.upper() 741 if self.conf.get(petscconfvar): 742 if isNull: 743 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 744 continue 745 continue # Success 746 elif not isNull: 747 if debug: print("requirement not found: ", requirement) 748 testDict['SKIP'].append(petscconfvar+" requirement not met") 749 continue 750 751 return testDict['SKIP'] == [] 752 753 def genPetscTests_summarize(self,dataDict): 754 """ 755 Required method to state what happened 756 """ 757 if not self.summarize: return 758 indent=" " 759 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 760 fh=open(fhname,"w") 761 #print "See ", fhname 762 for root in dataDict: 763 relroot=self.srcrelpath(root) 764 pkg=relroot.split("/")[1] 765 fh.write(relroot+"\n") 766 allSrcs=[] 767 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 768 for exfile in dataDict[root]: 769 # Basic information 770 rfile=os.path.join(relroot,exfile) 771 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 772 fh.write(indent+exfile+indent*4+builtStatus+"\n") 773 774 for test in dataDict[root][exfile]: 775 if test in self.buildkeys: continue 776 line=indent*2+test 777 fh.write(line+"\n") 778 # Looks nice to have the keys in order 779 #for key in dataDict[root][exfile][test]: 780 for key in "isrun abstracted nsize args requires script".split(): 781 if key not in dataDict[root][exfile][test]: continue 782 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 783 fh.write(line+"\n") 784 fh.write("\n") 785 fh.write("\n") 786 fh.write("\n") 787 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 788 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 789 fh.close() 790 return 791 792 def genPetscTests(self,root,dirs,files,dataDict): 793 """ 794 Go through and parse the source files in the directory to generate 795 the examples based on the metadata contained in the source files 796 """ 797 debug=False 798 # Use examplesAnalyze to get what the makefles think are sources 799 #self.examplesAnalyze(root,dirs,files,anlzDict) 800 801 dataDict[root]={} 802 803 for exfile in files: 804 #TST: Until we replace files, still leaving the orginals as is 805 #if not exfile.startswith("new_"+"ex"): continue 806 #if not exfile.startswith("ex"): continue 807 808 # Ignore emacs and other temporary files 809 if exfile.startswith("."): continue 810 if exfile.startswith("#"): continue 811 if exfile.endswith("~"): continue 812 813 # Convenience 814 fullex=os.path.join(root,exfile) 815 if self.verbose: print(' --> '+fullex) 816 dataDict[root].update(testparse.parseTestFile(fullex,0)) 817 if exfile in dataDict[root]: 818 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 819 820 return 821 822 def walktree(self,top): 823 """ 824 Walk a directory tree, starting from 'top' 825 """ 826 #print "action", action 827 # Goal of action is to fill this dictionary 828 dataDict={} 829 for root, dirs, files in os.walk(top, topdown=True): 830 dirs.sort() 831 files.sort() 832 if not "examples" in root: continue 833 if "dSYM" in root: continue 834 if os.path.basename(root.rstrip("/")) == 'output': continue 835 if self.verbose: print(root) 836 self.genPetscTests(root,dirs,files,dataDict) 837 # Now summarize this dictionary 838 self.genPetscTests_summarize(dataDict) 839 return dataDict 840 841 def gen_gnumake(self, fd): 842 """ 843 Overwrite of the method in the base PETSc class 844 """ 845 def write(stem, srcs): 846 for lang in LANGS: 847 if srcs[lang]['srcs']: 848 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 849 for pkg in PKGS: 850 srcs = self.gen_pkg(pkg) 851 write('testsrcs-' + pkg, srcs) 852 # Handle dependencies 853 for lang in LANGS: 854 for exfile in srcs[lang]['srcs']: 855 if exfile in srcs[lang]: 856 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 857 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 858 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 859 if deps: 860 # The executable literally depends on the object file because it is linked 861 fd.write(ex +": " + " ".join(deps) +'\n') 862 # The object file containing 'main' does not normally depend on other object 863 # files, but it does when it includes their modules. This dependency is 864 # overly blunt and could be reduced to only depend on object files for 865 # modules that are used, like "*f90aux.o". 866 fd.write(exfo +": " + " ".join(deps) +'\n') 867 868 return self.gendeps 869 870 def gen_pkg(self, pkg): 871 """ 872 Overwrite of the method in the base PETSc class 873 """ 874 return self.sources[pkg] 875 876 def write_gnumake(self, dataDict, output=None): 877 """ 878 Write out something similar to files from gmakegen.py 879 880 Test depends on script which also depends on source 881 file, but since I don't have a good way generating 882 acting on a single file (oops) just depend on 883 executable which in turn will depend on src file 884 """ 885 # Different options for how to set up the targets 886 compileExecsFirst=False 887 888 # Open file 889 fd = open(output, 'w') 890 891 # Write out the sources 892 gendeps = self.gen_gnumake(fd) 893 894 # Write out the tests and execname targets 895 fd.write("\n#Tests and executables\n") # Delimiter 896 897 for pkg in PKGS: 898 # These grab the ones that are built 899 for lang in LANGS: 900 testdeps=[] 901 for ftest in self.tests[pkg][lang]: 902 test=os.path.basename(ftest) 903 basedir=os.path.dirname(ftest) 904 testdeps.append(self.nameSpace(test,basedir)) 905 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 906 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 907 908 # test targets 909 for ftest in self.tests[pkg][lang]: 910 test=os.path.basename(ftest) 911 basedir=os.path.dirname(ftest) 912 testdir="${TESTDIR}/"+basedir+"/" 913 nmtest=self.nameSpace(test,basedir) 914 rundir=os.path.join(testdir,test) 915 #print test, nmtest 916 script=test+".sh" 917 918 # Deps 919 exfile=self.tests[pkg][lang][ftest]['exfile'] 920 fullex=os.path.join(self.srcdir,exfile) 921 localexec=self.tests[pkg][lang][ftest]['exec'] 922 execname=os.path.join(testdir,localexec) 923 fullscript=os.path.join(testdir,script) 924 tmpfile=os.path.join(testdir,test,test+".tmp") 925 926 # *.counts depends on the script and either executable (will 927 # be run) or the example source file (SKIP or TODO) 928 fd.write('%s.counts : %s %s' 929 % (os.path.join('$(TESTDIR)/counts', nmtest), 930 fullscript, 931 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 932 ) 933 if exfile in self.sources[pkg][lang]: 934 for dep in self.sources[pkg][lang][exfile]: 935 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 936 fd.write('\n') 937 938 # Now write the args: 939 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 940 941 fd.close() 942 return 943 944def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 945 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 946 if petsc_arch: 947 if len(petsc_arch.split(os.path.sep))>1: 948 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 949 output = os.path.join(testdir, 'testfiles') 950 951 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 952 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 953 testdir=testdir) 954 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 955 pEx.write_gnumake(dataDict, output) 956 957if __name__ == '__main__': 958 import optparse 959 parser = optparse.OptionParser() 960 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 961 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 962 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 963 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 964 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 965 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 966 967 opts, extra_args = parser.parse_args() 968 if extra_args: 969 import sys 970 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 971 exit(1) 972 if opts.testdir is None: 973 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 974 975 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 976 verbose=opts.verbose, 977 single_ex=opts.single_executable, srcdir=opts.srcdir, 978 testdir=opts.testdir) 979