1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs=[] 206 lsuffix='_' 207 argregex = re.compile(' (?=-[a-zA-Z])') 208 from testparse import parseLoopArgs 209 for key in inDict: 210 if key in ('SKIP', 'regexes'): 211 continue 212 akey=('subargs' if key=='args' else key) # what to assign 213 if akey not in inDict: inDict[akey]='' 214 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 215 # Always generate a loop over nsize, even if there is only one value 216 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 217 keystr = str(inDict[key]) 218 varlist = [] 219 for varset in argregex.split(keystr): 220 if not varset.strip(): continue 221 if '{{' in varset: 222 keyvar,lvars,ftype=parseLoopArgs(varset) 223 if akey not in loopVars: loopVars[akey]={} 224 varlist.append(keyvar) 225 loopVars[akey][keyvar]=[keyvar,lvars] 226 if akey=='nsize': 227 if len(lvars.split()) > 1: 228 lsuffix += akey +'-${' + keyvar + '}' 229 else: 230 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 231 lsuffix+=keyvar+'-${' + keyvar + '}_' 232 else: 233 if key=='args': 234 newargs.append(varset.strip()) 235 if varlist: 236 loopVars[akey]['varlist']=varlist 237 238 # For subtests, args are always substituted in (not top level) 239 if isSubtest: 240 inDict['subargs'] += " ".join(newargs) 241 inDict['args']='' 242 if 'label_suffix' in inDict: 243 inDict['label_suffix']+=lsuffix.rstrip('_') 244 else: 245 inDict['label_suffix']=lsuffix.rstrip('_') 246 else: 247 if loopVars: 248 inDict['args'] = ' '.join(newargs) 249 inDict['label_suffix']=lsuffix.rstrip('_') 250 return loopVars 251 252 def getArgLabel(self,testDict): 253 """ 254 In all of the arguments in the test dictionary, create a simple 255 string for searching within the makefile system. For simplicity in 256 search, remove "-", for strings, etc. 257 Also, concatenate the arg commands 258 For now, ignore nsize -- seems hard to search for anyway 259 """ 260 # Collect all of the args associated with a test 261 argStr=("" if 'args' not in testDict else testDict['args']) 262 if 'subtests' in testDict: 263 for stest in testDict["subtests"]: 264 sd=testDict[stest] 265 argStr=argStr+("" if 'args' not in sd else sd['args']) 266 267 # Now go through and cleanup 268 argStr=re.sub('{{(.*?)}}',"",argStr) 269 argStr=re.sub('-'," ",argStr) 270 for digit in string.digits: argStr=re.sub(digit," ",argStr) 271 argStr=re.sub("\.","",argStr) 272 argStr=re.sub(",","",argStr) 273 argStr=re.sub('\+',' ',argStr) 274 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 275 return argStr.strip() 276 277 def addToSources(self,exfile,rpath,srcDict): 278 """ 279 Put into data structure that allows easy generation of makefile 280 """ 281 pkg=rpath.split(os.path.sep)[0] 282 relpfile=os.path.join(rpath,exfile) 283 lang=self.getLanguage(exfile) 284 if not lang: return 285 self.sources[pkg][lang]['srcs'].append(relpfile) 286 self.sources[pkg][lang][relpfile] = [] 287 if 'depends' in srcDict: 288 depSrcList=srcDict['depends'].split() 289 for depSrc in depSrcList: 290 depObj=os.path.splitext(depSrc)[0]+".o" 291 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 292 293 # In gmakefile, ${TESTDIR} var specifies the object compilation 294 testsdir=rpath+"/" 295 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 296 self.objects[pkg].append(objfile) 297 return 298 299 def addToTests(self,test,rpath,exfile,execname,testDict): 300 """ 301 Put into data structure that allows easy generation of makefile 302 Organized by languages to allow testing of languages 303 """ 304 pkg=rpath.split("/")[0] 305 nmtest=os.path.join(rpath,test) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,rpath): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 if self.single_ex: 320 execname=rpath.split("/")[1]+"-ex" 321 else: 322 execname=os.path.splitext(exfile)[0] 323 return execname 324 325 def getSubstVars(self,testDict,rpath,testname): 326 """ 327 Create a dictionary with all of the variables that get substituted 328 into the template commands found in example_template.py 329 """ 330 subst={} 331 332 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 333 if 'nsize' not in testDict: testDict['nsize'] = '1' 334 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 335 for ak in testparse.acceptedkeys: 336 if ak=='test': continue 337 subst[ak]=(testDict[ak] if ak in testDict else '') 338 339 # Now do other variables 340 subst['execname']=testDict['execname'] 341 if 'filter' in testDict: 342 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 343 344 # Others 345 subst['subargs']='' # Default. For variables override 346 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 347 subst['label_suffix']='' 348 subst['comments']="\n#".join(subst['comments'].split("\n")) 349 if subst['comments']: subst['comments']="#"+subst['comments'] 350 subst['exec']="../"+subst['execname'] 351 subst['testroot']=self.testroot_dir 352 subst['testname']=testname 353 dp = self.conf.get('DATAFILESPATH','') 354 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 355 356 # This is used to label some matrices 357 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 358 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 359 360 #Conf vars 361 if self.petsc_arch.find('valgrind')>=0: 362 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 363 else: 364 subst['mpiexec']=self.conf['MPIEXEC'] 365 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 366 subst['petsc_arch']=self.petsc_arch 367 if self.inInstallDir: 368 # Case 2 369 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 370 else: 371 # Case 1 372 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 373 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 374 subst['diff']=self.conf['DIFF'] 375 subst['rm']=self.conf['RM'] 376 subst['grep']=self.conf['GREP'] 377 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 378 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 379 380 # Output file is special because of subtests override 381 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 382 if not "_" in defroot: defroot=defroot+"_1" 383 subst['defroot']=defroot 384 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 385 subst['redirect_file']=defroot+".tmp" 386 if 'output_file' not in testDict: 387 subst['output_file']="output/"+defroot+".out" 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/examples/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if not subst['filter_output']: 447 if 'altfiles' not in subst: 448 cmd=diffindnt+self._substVars(subst,example_template.difftest) 449 else: 450 # Have to do it by hand a bit because of variable number of alt files 451 rf=subst['redirect_file'] 452 cmd=diffindnt+example_template.difftest.split('@')[0] 453 for i in range(len(subst['altfiles'])): 454 af=subst['altfiles'][i] 455 cmd+=af+' '+rf 456 if i!=len(subst['altfiles'])-1: 457 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 458 cmd+=' || ${diff_exe} ' 459 else: 460 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 461 cmd+=subst['label_suffix']+' ""' # Quotes are painful 462 else: 463 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 464 cmdLines+=cmd+"\n" 465 cmdLines+=cmdindnt+'else\n' 466 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 467 cmdLines+=cmdindnt+'fi\n' 468 return cmdLines 469 470 def _writeTodoSkip(self,fh,tors,reasons,footer): 471 """ 472 Write out the TODO and SKIP lines in the file 473 The TODO or SKIP variable, tors, should be lower case 474 """ 475 TORS=tors.upper() 476 template=eval("example_template."+tors+"line") 477 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 478 tab = '' 479 if reasons: 480 fh.write('if ! $force; then\n') 481 tab = tab + ' ' 482 if reasons == ["Requires DATAFILESPATH"]: 483 # The only reason not to run is DATAFILESPATH, which we check at run-time 484 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 485 tab = tab + ' ' 486 if reasons: 487 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 488 fh.write(tab+footer+"\n") 489 fh.write(tab+"exit\n") 490 if reasons == ["Requires DATAFILESPATH"]: 491 fh.write(' fi\n') 492 if reasons: 493 fh.write('fi\n') 494 fh.write('\n\n') 495 return 496 497 def getLoopVarsHead(self,loopVars,i): 498 """ 499 Generate a nicely indented string with the format loops 500 Here is what the data structure looks like 501 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 502 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 503 loopVars['subargs']['pc_type']=["j","cholesky sor"] 504 """ 505 outstr=''; indnt=self.indent 506 for key in loopVars: 507 for var in loopVars[key]['varlist']: 508 varval=loopVars[key][var] 509 outstr += indnt * i + "for {0} in ${{{0}:-{1}}}; do\n".format(*varval) 510 i = i + 1 511 return (outstr,i) 512 513 def getLoopVarsFoot(self,loopVars,i): 514 outstr=''; indnt=self.indent 515 for key in loopVars: 516 for var in loopVars[key]['varlist']: 517 i = i - 1 518 outstr += indnt * i + "done\n" 519 return (outstr,i) 520 521 def genRunScript(self,testname,root,isRun,srcDict): 522 """ 523 Generate bash script using template found next to this file. 524 This file is read in at constructor time to avoid file I/O 525 """ 526 # runscript_dir directory has to be consistent with gmakefile 527 testDict=srcDict[testname] 528 rpath=self.srcrelpath(root) 529 runscript_dir=os.path.join(self.testroot_dir,rpath) 530 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 531 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 532 533 # Get variables to go into shell scripts. last time testDict used 534 subst=self.getSubstVars(testDict,rpath,testname) 535 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 536 if 'subtests' in testDict: 537 # The subtests inherit inDict, so we don't need top-level loops. 538 loopVars = {} 539 540 #Handle runfiles 541 for lfile in subst.get('localrunfiles','').split(): 542 install_files(os.path.join(root, lfile), 543 os.path.join(runscript_dir, os.path.dirname(lfile))) 544 # Check subtests for local runfiles 545 for stest in subst.get("subtests",[]): 546 for lfile in testDict[stest].get('localrunfiles','').split(): 547 install_files(os.path.join(root, lfile), 548 os.path.join(runscript_dir, os.path.dirname(lfile))) 549 550 # Now substitute the key variables into the header and footer 551 header=self._substVars(subst,example_template.header) 552 # The header is done twice to enable @...@ in header 553 header=self._substVars(subst,header) 554 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 555 556 # Start writing the file 557 fh.write(header+"\n") 558 559 # If there is a TODO or a SKIP then we do it before writing out the 560 # rest of the command (which is useful for working on the test) 561 # SKIP and TODO can be for the source file or for the runs 562 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 563 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 564 565 j=0 # for indentation 566 567 if loopVars: 568 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 569 if (loopHead): fh.write(loopHead+"\n") 570 571 # Subtests are special 572 if 'subtests' in testDict: 573 substP=subst # Subtests can inherit args but be careful 574 k=0 # for label suffixes 575 for stest in testDict["subtests"]: 576 subst=substP.copy() 577 subst.update(testDict[stest]) 578 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 579 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 580 if sLoopVars: 581 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 582 fh.write(sLoopHead+"\n") 583 fh.write(self.getCmds(subst,j)+"\n") 584 if sLoopVars: 585 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 586 fh.write(sLoopFoot+"\n") 587 else: 588 fh.write(self.getCmds(subst,j)+"\n") 589 590 if loopVars: 591 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 592 fh.write(loopFoot+"\n") 593 594 fh.write(footer+"\n") 595 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 596 #if '10_9' in testname: sys.exit() 597 return 598 599 def genScriptsAndInfo(self,exfile,root,srcDict): 600 """ 601 Generate scripts from the source file, determine if built, etc. 602 For every test in the exfile with info in the srcDict: 603 1. Determine if it needs to be run for this arch 604 2. Generate the script 605 3. Generate the data needed to write out the makefile in a 606 convenient way 607 All tests are *always* run, but some may be SKIP'd per the TAP standard 608 """ 609 debug=False 610 rpath=self.srcrelpath(root) 611 execname=self.getExecname(exfile,rpath) 612 isBuilt=self._isBuilt(exfile,srcDict) 613 for test in srcDict: 614 if test in self.buildkeys: continue 615 if debug: print(self.nameSpace(exfile,root), test) 616 srcDict[test]['execname']=execname # Convenience in generating scripts 617 isRun=self._isRun(srcDict[test]) 618 self.genRunScript(test,root,isRun,srcDict) 619 srcDict[test]['isrun']=isRun 620 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 621 622 # This adds to datastructure for building deps 623 if isBuilt: self.addToSources(exfile,rpath,srcDict) 624 return 625 626 def _isBuilt(self,exfile,srcDict): 627 """ 628 Determine if this file should be built. 629 """ 630 # Get the language based on file extension 631 srcDict['SKIP'] = [] 632 lang=self.getLanguage(exfile) 633 if (lang=="F" or lang=="F90"): 634 if not self.have_fortran: 635 srcDict["SKIP"].append("Fortran required for this test") 636 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 637 srcDict["SKIP"].append("Fortran f90freeform required for this test") 638 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 639 srcDict["SKIP"].append("CUDA required for this test") 640 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 641 srcDict["SKIP"].append("C++ required for this test") 642 643 # Deprecated source files 644 if srcDict.get("TODO"): 645 return False 646 647 # isRun can work with srcDict to handle the requires 648 if "requires" in srcDict: 649 if srcDict["requires"]: 650 return self._isRun(srcDict) 651 652 return srcDict['SKIP'] == [] 653 654 655 def _isRun(self,testDict, debug=False): 656 """ 657 Based on the requirements listed in the src file and the petscconf.h 658 info, determine whether this test should be run or not. 659 """ 660 indent=" " 661 662 if 'SKIP' not in testDict: 663 testDict['SKIP'] = [] 664 # MPI requirements 665 if 'MPI_IS_MPIUNI' in self.conf: 666 if testDict.get('nsize', '1') != '1': 667 testDict['SKIP'].append("Parallel test with serial build") 668 669 # The requirements for the test are the sum of all the run subtests 670 if 'subtests' in testDict: 671 if 'requires' not in testDict: testDict['requires']="" 672 for stest in testDict['subtests']: 673 if 'requires' in testDict[stest]: 674 testDict['requires']+=" "+testDict[stest]['requires'] 675 if testDict.get('nsize', '1') != '1': 676 testDict['SKIP'].append("Parallel test with serial build") 677 break 678 679 # Now go through all requirements 680 if 'requires' in testDict: 681 for requirement in testDict['requires'].split(): 682 requirement=requirement.strip() 683 if not requirement: continue 684 if debug: print(indent+"Requirement: ", requirement) 685 isNull=False 686 if requirement.startswith("!"): 687 requirement=requirement[1:]; isNull=True 688 # Precision requirement for reals 689 if requirement in self.precision_types: 690 if self.conf['PETSC_PRECISION']==requirement: 691 if isNull: 692 testDict['SKIP'].append("not "+requirement+" required") 693 continue 694 continue # Success 695 elif not isNull: 696 testDict['SKIP'].append(requirement+" required") 697 continue 698 # Precision requirement for ints 699 if requirement in self.integer_types: 700 if requirement=="int32": 701 if self.conf['PETSC_SIZEOF_INT']==4: 702 if isNull: 703 testDict['SKIP'].append("not int32 required") 704 continue 705 continue # Success 706 elif not isNull: 707 testDict['SKIP'].append("int32 required") 708 continue 709 if requirement=="int64": 710 if self.conf['PETSC_SIZEOF_INT']==8: 711 if isNull: 712 testDict['SKIP'].append("NOT int64 required") 713 continue 714 continue # Success 715 elif not isNull: 716 testDict['SKIP'].append("int64 required") 717 continue 718 # Datafilespath 719 if requirement=="datafilespath" and not isNull: 720 testDict['SKIP'].append("Requires DATAFILESPATH") 721 continue 722 # Defines -- not sure I have comments matching 723 if "define(" in requirement.lower(): 724 reqdef=requirement.split("(")[1].split(")")[0] 725 if reqdef in self.conf: 726 if isNull: 727 testDict['SKIP'].append("Null requirement not met: "+requirement) 728 continue 729 continue # Success 730 elif not isNull: 731 testDict['SKIP'].append("Required: "+requirement) 732 continue 733 734 # Rest should be packages that we can just get from conf 735 if requirement == "complex": 736 petscconfvar="PETSC_USE_COMPLEX" 737 else: 738 petscconfvar="PETSC_HAVE_"+requirement.upper() 739 if self.conf.get(petscconfvar): 740 if isNull: 741 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 742 continue 743 continue # Success 744 elif not isNull: 745 if debug: print("requirement not found: ", requirement) 746 testDict['SKIP'].append(petscconfvar+" requirement not met") 747 continue 748 749 return testDict['SKIP'] == [] 750 751 def genPetscTests_summarize(self,dataDict): 752 """ 753 Required method to state what happened 754 """ 755 if not self.summarize: return 756 indent=" " 757 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 758 fh=open(fhname,"w") 759 for root in dataDict: 760 relroot=self.srcrelpath(root) 761 pkg=relroot.split("/")[1] 762 fh.write(relroot+"\n") 763 allSrcs=[] 764 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 765 for exfile in dataDict[root]: 766 # Basic information 767 rfile=os.path.join(relroot,exfile) 768 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 769 fh.write(indent+exfile+indent*4+builtStatus+"\n") 770 771 for test in dataDict[root][exfile]: 772 if test in self.buildkeys: continue 773 line=indent*2+test 774 fh.write(line+"\n") 775 # Looks nice to have the keys in order 776 #for key in dataDict[root][exfile][test]: 777 for key in "isrun abstracted nsize args requires script".split(): 778 if key not in dataDict[root][exfile][test]: continue 779 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 780 fh.write(line+"\n") 781 fh.write("\n") 782 fh.write("\n") 783 fh.write("\n") 784 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 785 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 786 fh.close() 787 return 788 789 def genPetscTests(self,root,dirs,files,dataDict): 790 """ 791 Go through and parse the source files in the directory to generate 792 the examples based on the metadata contained in the source files 793 """ 794 debug=False 795 # Use examplesAnalyze to get what the makefles think are sources 796 #self.examplesAnalyze(root,dirs,files,anlzDict) 797 798 dataDict[root]={} 799 800 for exfile in files: 801 #TST: Until we replace files, still leaving the orginals as is 802 #if not exfile.startswith("new_"+"ex"): continue 803 #if not exfile.startswith("ex"): continue 804 805 # Ignore emacs and other temporary files 806 if exfile.startswith("."): continue 807 if exfile.startswith("#"): continue 808 if exfile.endswith("~"): continue 809 # Only parse source files 810 ext=os.path.splitext(exfile)[-1].lstrip('.') 811 if ext not in LANGS: continue 812 813 # Convenience 814 fullex=os.path.join(root,exfile) 815 if self.verbose: print(' --> '+fullex) 816 dataDict[root].update(testparse.parseTestFile(fullex,0)) 817 if exfile in dataDict[root]: 818 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 819 820 return 821 822 def walktree(self,top): 823 """ 824 Walk a directory tree, starting from 'top' 825 """ 826 # Goal of action is to fill this dictionary 827 dataDict={} 828 for root, dirs, files in os.walk(top, topdown=True): 829 dirs.sort() 830 files.sort() 831 if not "examples" in root: continue 832 if "dSYM" in root: continue 833 if os.path.basename(root.rstrip("/")) == 'output': continue 834 if self.verbose: print(root) 835 self.genPetscTests(root,dirs,files,dataDict) 836 # Now summarize this dictionary 837 if self.verbose: self.genPetscTests_summarize(dataDict) 838 return dataDict 839 840 def gen_gnumake(self, fd): 841 """ 842 Overwrite of the method in the base PETSc class 843 """ 844 def write(stem, srcs): 845 for lang in LANGS: 846 if srcs[lang]['srcs']: 847 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 848 for pkg in PKGS: 849 srcs = self.gen_pkg(pkg) 850 write('testsrcs-' + pkg, srcs) 851 # Handle dependencies 852 for lang in LANGS: 853 for exfile in srcs[lang]['srcs']: 854 if exfile in srcs[lang]: 855 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 856 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 857 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 858 if deps: 859 # The executable literally depends on the object file because it is linked 860 fd.write(ex +": " + " ".join(deps) +'\n') 861 # The object file containing 'main' does not normally depend on other object 862 # files, but it does when it includes their modules. This dependency is 863 # overly blunt and could be reduced to only depend on object files for 864 # modules that are used, like "*f90aux.o". 865 fd.write(exfo +": " + " ".join(deps) +'\n') 866 867 return self.gendeps 868 869 def gen_pkg(self, pkg): 870 """ 871 Overwrite of the method in the base PETSc class 872 """ 873 return self.sources[pkg] 874 875 def write_gnumake(self, dataDict, output=None): 876 """ 877 Write out something similar to files from gmakegen.py 878 879 Test depends on script which also depends on source 880 file, but since I don't have a good way generating 881 acting on a single file (oops) just depend on 882 executable which in turn will depend on src file 883 """ 884 # Different options for how to set up the targets 885 compileExecsFirst=False 886 887 # Open file 888 fd = open(output, 'w') 889 890 # Write out the sources 891 gendeps = self.gen_gnumake(fd) 892 893 # Write out the tests and execname targets 894 fd.write("\n#Tests and executables\n") # Delimiter 895 896 for pkg in PKGS: 897 # These grab the ones that are built 898 for lang in LANGS: 899 testdeps=[] 900 for ftest in self.tests[pkg][lang]: 901 test=os.path.basename(ftest) 902 basedir=os.path.dirname(ftest) 903 testdeps.append(self.nameSpace(test,basedir)) 904 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 905 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 906 907 # test targets 908 for ftest in self.tests[pkg][lang]: 909 test=os.path.basename(ftest) 910 basedir=os.path.dirname(ftest) 911 testdir="${TESTDIR}/"+basedir+"/" 912 nmtest=self.nameSpace(test,basedir) 913 rundir=os.path.join(testdir,test) 914 script=test+".sh" 915 916 # Deps 917 exfile=self.tests[pkg][lang][ftest]['exfile'] 918 fullex=os.path.join(self.srcdir,exfile) 919 localexec=self.tests[pkg][lang][ftest]['exec'] 920 execname=os.path.join(testdir,localexec) 921 fullscript=os.path.join(testdir,script) 922 tmpfile=os.path.join(testdir,test,test+".tmp") 923 924 # *.counts depends on the script and either executable (will 925 # be run) or the example source file (SKIP or TODO) 926 fd.write('%s.counts : %s %s' 927 % (os.path.join('$(TESTDIR)/counts', nmtest), 928 fullscript, 929 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 930 ) 931 if exfile in self.sources[pkg][lang]: 932 for dep in self.sources[pkg][lang][exfile]: 933 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 934 fd.write('\n') 935 936 # Now write the args: 937 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 938 939 fd.close() 940 return 941 942def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 943 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 944 if petsc_arch: 945 if len(petsc_arch.split(os.path.sep))>1: 946 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 947 output = os.path.join(testdir, 'testfiles') 948 949 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 950 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 951 testdir=testdir) 952 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 953 pEx.write_gnumake(dataDict, output) 954 955if __name__ == '__main__': 956 import optparse 957 parser = optparse.OptionParser() 958 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 959 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 960 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 961 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 962 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 963 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 964 965 opts, extra_args = parser.parse_args() 966 if extra_args: 967 import sys 968 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 969 exit(1) 970 if opts.testdir is None: 971 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 972 973 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 974 verbose=opts.verbose, 975 single_ex=opts.single_executable, srcdir=opts.srcdir, 976 testdir=opts.testdir) 977