1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from collections import defaultdict 10from gmakegen import * 11 12import inspect 13thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 14sys.path.insert(0,thisscriptdir) 15import testparse 16import example_template 17 18 19""" 20 21There are 2 modes of running tests: Normal builds and run from prefix of 22install. They affect where to find things: 23 24 25Case 1. Normal builds: 26 27 +---------------------+----------------------------------+ 28 | PETSC_DIR | <git dir> | 29 +---------------------+----------------------------------+ 30 | PETSC_ARCH | arch-foo | 31 +---------------------+----------------------------------+ 32 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 33 +---------------------+----------------------------------+ 34 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 35 +---------------------+----------------------------------+ 36 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 37 +---------------------+----------------------------------+ 38 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 41 +---------------------+----------------------------------+ 42 43 44Case 2. From install dir: 45 46 +---------------------+-------------------------------------------------------+ 47 | PETSC_DIR | <prefix dir> | 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_ARCH | '' | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 60 +---------------------+-------------------------------------------------------+ 61 62""" 63 64def install_files(source, destdir): 65 """Install file or directory 'source' to 'destdir'. Does not preserve 66 mode (permissions). 67 """ 68 if not os.path.isdir(destdir): 69 os.makedirs(destdir) 70 if os.path.isdir(source): 71 for name in os.listdir(source): 72 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 73 else: 74 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 75 76class generateExamples(Petsc): 77 """ 78 gmakegen.py has basic structure for finding the files, writing out 79 the dependencies, etc. 80 """ 81 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 82 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 83 84 self.single_ex=single_ex 85 self.srcdir=srcdir 86 self.check_output=check 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 if self.srcdir is None: 98 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 99 else: 100 if petsc_arch == '': 101 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 102 # Case 1 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 if self.srcdir is None: 105 self.srcdir=os.path.join(self.petsc_dir,'src') 106 107 self.testroot_dir=os.path.abspath(testdir) 108 109 self.ptNaming=True 110 self.verbose=verbose 111 # Whether to write out a useful debugging 112 self.summarize=True if verbose else False 113 114 # For help in setting the requirements 115 self.precision_types="single double __float128 int32".split() 116 self.integer_types="int32 int64 long32 long64".split() 117 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 118 119 # Things that are not test 120 self.buildkeys=testparse.buildkeys 121 122 # Adding a dictionary for storing sources, objects, and tests 123 # to make building the dependency tree easier 124 self.sources={} 125 self.objects={} 126 self.tests={} 127 for pkg in self.pkg_pkgs: 128 self.sources[pkg]={} 129 self.objects[pkg]=[] 130 self.tests[pkg]={} 131 for lang in LANGS: 132 self.sources[pkg][lang]={} 133 self.sources[pkg][lang]['srcs']=[] 134 self.tests[pkg][lang]={} 135 136 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 137 138 self.indent=" " 139 if self.verbose: print('Finishing the constructor') 140 return 141 142 def srcrelpath(self,rdir): 143 """ 144 Get relative path to source directory 145 """ 146 return os.path.relpath(rdir,self.srcdir) 147 148 def getInInstallDir(self,thisscriptdir): 149 """ 150 When petsc is installed then this file in installed in: 151 <PREFIX>/share/petsc/examples/config/gmakegentest.py 152 otherwise the path is: 153 <PETSC_DIR>/config/gmakegentest.py 154 We use this difference to determine if we are in installdir 155 """ 156 dirlist=thisscriptdir.split(os.path.sep) 157 if len(dirlist)>4: 158 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 159 if lastfour==os.path.join('share','petsc','examples','config'): 160 return True 161 else: 162 return False 163 else: 164 return False 165 166 def nameSpace(self,srcfile,srcdir): 167 """ 168 Because the scripts have a non-unique naming, the pretty-printing 169 needs to convey the srcdir and srcfile. There are two ways of doing this. 170 """ 171 if self.ptNaming: 172 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 173 cdir=srcdir 174 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 175 nameString=prefix+srcfile 176 else: 177 #nameString=srcdir+": "+srcfile 178 nameString=srcfile 179 return nameString 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:\] now? 185 """ 186 langReq=None 187 srcext=os.path.splitext(srcfile)[-1] 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " "+" ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 if pkg not in self.sources: return 289 self.sources[pkg][lang]['srcs'].append(relpfile) 290 self.sources[pkg][lang][relpfile] = [] 291 if 'depends' in srcDict: 292 depSrcList=srcDict['depends'].split() 293 for depSrc in depSrcList: 294 depObj=os.path.splitext(depSrc)[0]+".o" 295 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 296 297 # In gmakefile, ${TESTDIR} var specifies the object compilation 298 testsdir=rpath+"/" 299 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 300 self.objects[pkg].append(objfile) 301 return 302 303 def addToTests(self,test,rpath,exfile,execname,testDict): 304 """ 305 Put into data structure that allows easy generation of makefile 306 Organized by languages to allow testing of languages 307 """ 308 pkg=rpath.split("/")[0] 309 nmtest=os.path.join(rpath,test) 310 lang=self.getLanguage(exfile) 311 if not lang: return 312 if pkg not in self.tests: return 313 self.tests[pkg][lang][nmtest]={} 314 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 315 self.tests[pkg][lang][nmtest]['exec']=execname 316 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 317 return 318 319 def getExecname(self,exfile,rpath): 320 """ 321 Generate bash script using template found next to this file. 322 This file is read in at constructor time to avoid file I/O 323 """ 324 if self.single_ex: 325 execname=rpath.split("/")[1]+"-ex" 326 else: 327 execname=os.path.splitext(exfile)[0] 328 return execname 329 330 def getSubstVars(self,testDict,rpath,testname): 331 """ 332 Create a dictionary with all of the variables that get substituted 333 into the template commands found in example_template.py 334 """ 335 subst={} 336 337 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 338 if 'nsize' not in testDict: testDict['nsize'] = '1' 339 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 340 for ak in testparse.acceptedkeys: 341 if ak=='test': continue 342 subst[ak]=(testDict[ak] if ak in testDict else '') 343 344 # Now do other variables 345 subst['execname']=testDict['execname'] 346 if 'filter' in testDict: 347 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(self.srcdir, rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['pkg_name']=self.pkg_name 371 subst['pkg_dir']=self.pkg_dir 372 subst['pkg_arch']=self.petsc_arch 373 subst['CONFIG_DIR']=thisscriptdir 374 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 375 subst['diff']=self.conf['DIFF'] 376 subst['rm']=self.conf['RM'] 377 subst['grep']=self.conf['GREP'] 378 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 379 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 380 381 # Output file is special because of subtests override 382 defroot = testparse.getDefaultOutputFileRoot(testname) 383 if 'output_file' not in testDict: 384 subst['output_file']="output/"+defroot+".out" 385 subst['redirect_file']=defroot+".tmp" 386 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 387 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/examples/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if not subst['filter_output']: 447 if 'altfiles' not in subst: 448 cmd=diffindnt+self._substVars(subst,example_template.difftest) 449 else: 450 # Have to do it by hand a bit because of variable number of alt files 451 rf=subst['redirect_file'] 452 cmd=diffindnt+example_template.difftest.split('@')[0] 453 for i in range(len(subst['altfiles'])): 454 af=subst['altfiles'][i] 455 cmd+=af+' '+rf 456 if i!=len(subst['altfiles'])-1: 457 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 458 cmd+=' || ${diff_exe} ' 459 else: 460 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 461 cmd+=subst['label_suffix']+' ""' # Quotes are painful 462 else: 463 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 464 cmdLines+=cmd+"\n" 465 cmdLines+=cmdindnt+'else\n' 466 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 467 cmdLines+=cmdindnt+'fi\n' 468 return cmdLines 469 470 def _writeTodoSkip(self,fh,tors,reasons,footer): 471 """ 472 Write out the TODO and SKIP lines in the file 473 The TODO or SKIP variable, tors, should be lower case 474 """ 475 TORS=tors.upper() 476 template=eval("example_template."+tors+"line") 477 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 478 tab = '' 479 if reasons: 480 fh.write('if ! $force; then\n') 481 tab = tab + ' ' 482 if reasons == ["Requires DATAFILESPATH"]: 483 # The only reason not to run is DATAFILESPATH, which we check at run-time 484 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 485 tab = tab + ' ' 486 if reasons: 487 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 488 fh.write(tab+footer+"\n") 489 fh.write(tab+"exit\n") 490 if reasons == ["Requires DATAFILESPATH"]: 491 fh.write(' fi\n') 492 if reasons: 493 fh.write('fi\n') 494 fh.write('\n\n') 495 return 496 497 def getLoopVarsHead(self,loopVars,i,usedVars={}): 498 """ 499 Generate a nicely indented string with the format loops 500 Here is what the data structure looks like 501 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 502 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 503 loopVars['subargs']['pc_type']=["j","cholesky sor"] 504 """ 505 outstr=''; indnt=self.indent 506 507 for key in loopVars: 508 if key in usedVars: continue # Do not duplicate setting vars 509 for var in loopVars[key]['varlist']: 510 varval=loopVars[key][var] 511 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 512 outstr += "\n\n" 513 514 for key in loopVars: 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 518 i = i + 1 519 return (outstr,i) 520 521 def getLoopVarsFoot(self,loopVars,i): 522 outstr=''; indnt=self.indent 523 for key in loopVars: 524 for var in loopVars[key]['varlist']: 525 i = i - 1 526 outstr += indnt * i + "done\n" 527 return (outstr,i) 528 529 def genRunScript(self,testname,root,isRun,srcDict): 530 """ 531 Generate bash script using template found next to this file. 532 This file is read in at constructor time to avoid file I/O 533 """ 534 # runscript_dir directory has to be consistent with gmakefile 535 testDict=srcDict[testname] 536 rpath=self.srcrelpath(root) 537 runscript_dir=os.path.join(self.testroot_dir,rpath) 538 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 539 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 540 541 # Get variables to go into shell scripts. last time testDict used 542 subst=self.getSubstVars(testDict,rpath,testname) 543 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 544 if 'subtests' in testDict: 545 # The subtests inherit inDict, so we don't need top-level loops. 546 loopVars = {} 547 548 #Handle runfiles 549 for lfile in subst.get('localrunfiles','').split(): 550 install_files(os.path.join(root, lfile), 551 os.path.join(runscript_dir, os.path.dirname(lfile))) 552 # Check subtests for local runfiles 553 for stest in subst.get("subtests",[]): 554 for lfile in testDict[stest].get('localrunfiles','').split(): 555 install_files(os.path.join(root, lfile), 556 os.path.join(runscript_dir, os.path.dirname(lfile))) 557 558 # Now substitute the key variables into the header and footer 559 header=self._substVars(subst,example_template.header) 560 # The header is done twice to enable @...@ in header 561 header=self._substVars(subst,header) 562 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 563 564 # Start writing the file 565 fh.write(header+"\n") 566 567 # If there is a TODO or a SKIP then we do it before writing out the 568 # rest of the command (which is useful for working on the test) 569 # SKIP and TODO can be for the source file or for the runs 570 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 571 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 572 573 j=0 # for indentation 574 575 if loopVars: 576 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 577 if (loopHead): fh.write(loopHead+"\n") 578 579 # Subtests are special 580 allLoopVars=list(loopVars.keys()) 581 if 'subtests' in testDict: 582 substP=subst # Subtests can inherit args but be careful 583 k=0 # for label suffixes 584 for stest in testDict["subtests"]: 585 subst=substP.copy() 586 subst.update(testDict[stest]) 587 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 588 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 589 if sLoopVars: 590 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 591 allLoopVars+=list(sLoopVars.keys()) 592 fh.write(sLoopHead+"\n") 593 fh.write(self.getCmds(subst,j)+"\n") 594 if sLoopVars: 595 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 596 fh.write(sLoopFoot+"\n") 597 else: 598 fh.write(self.getCmds(subst,j)+"\n") 599 600 if loopVars: 601 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 602 fh.write(loopFoot+"\n") 603 604 fh.write(footer+"\n") 605 fh.close() 606 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 607 #if '10_9' in testname: sys.exit() 608 return 609 610 def genScriptsAndInfo(self,exfile,root,srcDict): 611 """ 612 Generate scripts from the source file, determine if built, etc. 613 For every test in the exfile with info in the srcDict: 614 1. Determine if it needs to be run for this arch 615 2. Generate the script 616 3. Generate the data needed to write out the makefile in a 617 convenient way 618 All tests are *always* run, but some may be SKIP'd per the TAP standard 619 """ 620 debug=False 621 rpath=self.srcrelpath(root) 622 execname=self.getExecname(exfile,rpath) 623 isBuilt=self._isBuilt(exfile,srcDict) 624 for test in srcDict: 625 if test in self.buildkeys: continue 626 if debug: print(self.nameSpace(exfile,root), test) 627 srcDict[test]['execname']=execname # Convenience in generating scripts 628 isRun=self._isRun(srcDict[test]) 629 self.genRunScript(test,root,isRun,srcDict) 630 srcDict[test]['isrun']=isRun 631 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 632 633 # This adds to datastructure for building deps 634 if isBuilt: self.addToSources(exfile,rpath,srcDict) 635 return 636 637 def _isBuilt(self,exfile,srcDict): 638 """ 639 Determine if this file should be built. 640 """ 641 # Get the language based on file extension 642 srcDict['SKIP'] = [] 643 lang=self.getLanguage(exfile) 644 if (lang=="F" or lang=="F90"): 645 if not self.have_fortran: 646 srcDict["SKIP"].append("Fortran required for this test") 647 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 648 srcDict["SKIP"].append("Fortran f90freeform required for this test") 649 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 650 srcDict["SKIP"].append("CUDA required for this test") 651 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 652 srcDict["SKIP"].append("C++ required for this test") 653 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 654 srcDict["SKIP"].append("C++ required for this test") 655 656 # Deprecated source files 657 if srcDict.get("TODO"): 658 return False 659 660 # isRun can work with srcDict to handle the requires 661 if "requires" in srcDict: 662 if srcDict["requires"]: 663 return self._isRun(srcDict) 664 665 return srcDict['SKIP'] == [] 666 667 668 def _isRun(self,testDict, debug=False): 669 """ 670 Based on the requirements listed in the src file and the petscconf.h 671 info, determine whether this test should be run or not. 672 """ 673 indent=" " 674 675 if 'SKIP' not in testDict: 676 testDict['SKIP'] = [] 677 # MPI requirements 678 if 'MPI_IS_MPIUNI' in self.conf: 679 if testDict.get('nsize', '1') != '1': 680 testDict['SKIP'].append("Parallel test with serial build") 681 682 # The requirements for the test are the sum of all the run subtests 683 if 'subtests' in testDict: 684 if 'requires' not in testDict: testDict['requires']="" 685 for stest in testDict['subtests']: 686 if 'requires' in testDict[stest]: 687 testDict['requires']+=" "+testDict[stest]['requires'] 688 if testDict.get('nsize', '1') != '1': 689 testDict['SKIP'].append("Parallel test with serial build") 690 break 691 692 # Now go through all requirements 693 if 'requires' in testDict: 694 for requirement in testDict['requires'].split(): 695 requirement=requirement.strip() 696 if not requirement: continue 697 if debug: print(indent+"Requirement: ", requirement) 698 isNull=False 699 if requirement.startswith("!"): 700 requirement=requirement[1:]; isNull=True 701 # Precision requirement for reals 702 if requirement in self.precision_types: 703 if self.conf['PETSC_PRECISION']==requirement: 704 if isNull: 705 testDict['SKIP'].append("not "+requirement+" required") 706 continue 707 continue # Success 708 elif not isNull: 709 testDict['SKIP'].append(requirement+" required") 710 continue 711 # Precision requirement for ints 712 if requirement in self.integer_types: 713 if requirement=="int32": 714 if self.conf['PETSC_SIZEOF_INT']==4: 715 if isNull: 716 testDict['SKIP'].append("not int32 required") 717 continue 718 continue # Success 719 elif not isNull: 720 testDict['SKIP'].append("int32 required") 721 continue 722 if requirement=="int64": 723 if self.conf['PETSC_SIZEOF_INT']==8: 724 if isNull: 725 testDict['SKIP'].append("NOT int64 required") 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append("int64 required") 730 continue 731 if requirement.startswith("long"): 732 reqsize = int(requirement[4:])//8 733 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 734 if longsize==reqsize: 735 if isNull: 736 testDict['SKIP'].append("not %s required" % requirement) 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("%s required" % requirement) 741 continue 742 # Datafilespath 743 if requirement=="datafilespath" and not isNull: 744 testDict['SKIP'].append("Requires DATAFILESPATH") 745 continue 746 # Defines -- not sure I have comments matching 747 if "define(" in requirement.lower(): 748 reqdef=requirement.split("(")[1].split(")")[0] 749 if reqdef in self.conf: 750 if isNull: 751 testDict['SKIP'].append("Null requirement not met: "+requirement) 752 continue 753 continue # Success 754 elif not isNull: 755 testDict['SKIP'].append("Required: "+requirement) 756 continue 757 758 # Rest should be packages that we can just get from conf 759 if requirement in ["complex","debug"]: 760 petscconfvar="PETSC_USE_"+requirement.upper() 761 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 762 else: 763 petscconfvar="PETSC_HAVE_"+requirement.upper() 764 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 765 petsccv = self.conf.get(petscconfvar) 766 pkgcv = self.conf.get(pkgconfvar) 767 768 if petsccv or pkgcv: 769 if isNull: 770 if petsccv: 771 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 772 continue 773 else: 774 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 775 continue 776 continue # Success 777 elif not isNull: 778 if not petsccv and not pkgcv: 779 if debug: print("requirement not found: ", requirement) 780 if self.pkg_name == 'petsc': 781 testDict['SKIP'].append(petscconfvar+" requirement not met") 782 else: 783 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 784 continue 785 return testDict['SKIP'] == [] 786 787 def checkOutput(self,exfile,root,srcDict): 788 """ 789 Check and make sure the output files are in the output director 790 """ 791 debug=False 792 rpath=self.srcrelpath(root) 793 for test in srcDict: 794 if test in self.buildkeys: continue 795 if debug: print(rpath, exfile, test) 796 if 'output_file' in srcDict[test]: 797 output_file=srcDict[test]['output_file'] 798 else: 799 defroot = testparse.getDefaultOutputFileRoot(test) 800 if 'TODO' in srcDict[test]: continue 801 output_file="output/"+defroot+".out" 802 803 fullout=os.path.join(root,output_file) 804 if debug: print("---> ",fullout) 805 if not os.path.exists(fullout): 806 self.missing_files.append(fullout) 807 808 return 809 810 def genPetscTests_summarize(self,dataDict): 811 """ 812 Required method to state what happened 813 """ 814 if not self.summarize: return 815 indent=" " 816 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 817 fh=open(fhname,"w") 818 for root in dataDict: 819 relroot=self.srcrelpath(root) 820 pkg=relroot.split("/")[1] 821 fh.write(relroot+"\n") 822 allSrcs=[] 823 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 824 for exfile in dataDict[root]: 825 # Basic information 826 rfile=os.path.join(relroot,exfile) 827 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 828 fh.write(indent+exfile+indent*4+builtStatus+"\n") 829 830 for test in dataDict[root][exfile]: 831 if test in self.buildkeys: continue 832 line=indent*2+test 833 fh.write(line+"\n") 834 # Looks nice to have the keys in order 835 #for key in dataDict[root][exfile][test]: 836 for key in "isrun abstracted nsize args requires script".split(): 837 if key not in dataDict[root][exfile][test]: continue 838 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 839 fh.write(line+"\n") 840 fh.write("\n") 841 fh.write("\n") 842 fh.write("\n") 843 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 844 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 845 fh.close() 846 return 847 848 def genPetscTests(self,root,dirs,files,dataDict): 849 """ 850 Go through and parse the source files in the directory to generate 851 the examples based on the metadata contained in the source files 852 """ 853 debug=False 854 # Use examplesAnalyze to get what the makefles think are sources 855 #self.examplesAnalyze(root,dirs,files,anlzDict) 856 857 dataDict[root]={} 858 859 for exfile in files: 860 #TST: Until we replace files, still leaving the orginals as is 861 #if not exfile.startswith("new_"+"ex"): continue 862 #if not exfile.startswith("ex"): continue 863 864 # Ignore emacs and other temporary files 865 if exfile.startswith("."): continue 866 if exfile.startswith("#"): continue 867 if exfile.endswith("~"): continue 868 # Only parse source files 869 ext=os.path.splitext(exfile)[-1].lstrip('.') 870 if ext not in LANGS: continue 871 872 # Convenience 873 fullex=os.path.join(root,exfile) 874 if self.verbose: print(' --> '+fullex) 875 dataDict[root].update(testparse.parseTestFile(fullex,0)) 876 if exfile in dataDict[root]: 877 if not self.check_output: 878 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 879 else: 880 self.checkOutput(exfile,root,dataDict[root][exfile]) 881 882 return 883 884 def walktree(self,top): 885 """ 886 Walk a directory tree, starting from 'top' 887 """ 888 if self.check_output: 889 print("Checking for missing output files") 890 self.missing_files=[] 891 892 # Goal of action is to fill this dictionary 893 dataDict={} 894 for root, dirs, files in os.walk(top, topdown=True): 895 dirs.sort() 896 files.sort() 897 if not "examples" in root: continue 898 if "dSYM" in root: continue 899 if os.path.basename(root.rstrip("/")) == 'output': continue 900 if self.verbose: print(root) 901 self.genPetscTests(root,dirs,files,dataDict) 902 903 # If checking output, report results 904 if self.check_output: 905 if self.missing_files: 906 for file in set(self.missing_files): # set uniqifies 907 print(file) 908 sys.exit(1) 909 910 # Now summarize this dictionary 911 if self.verbose: self.genPetscTests_summarize(dataDict) 912 return dataDict 913 914 def gen_gnumake(self, fd): 915 """ 916 Overwrite of the method in the base PETSc class 917 """ 918 def write(stem, srcs): 919 for lang in LANGS: 920 if srcs[lang]['srcs']: 921 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 922 for pkg in self.pkg_pkgs: 923 srcs = self.gen_pkg(pkg) 924 write('testsrcs-' + pkg, srcs) 925 # Handle dependencies 926 for lang in LANGS: 927 for exfile in srcs[lang]['srcs']: 928 if exfile in srcs[lang]: 929 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 930 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 931 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 932 if deps: 933 # The executable literally depends on the object file because it is linked 934 fd.write(ex +": " + " ".join(deps) +'\n') 935 # The object file containing 'main' does not normally depend on other object 936 # files, but it does when it includes their modules. This dependency is 937 # overly blunt and could be reduced to only depend on object files for 938 # modules that are used, like "*f90aux.o". 939 fd.write(exfo +": " + " ".join(deps) +'\n') 940 941 return self.gendeps 942 943 def gen_pkg(self, pkg): 944 """ 945 Overwrite of the method in the base PETSc class 946 """ 947 return self.sources[pkg] 948 949 def write_gnumake(self, dataDict, output=None): 950 """ 951 Write out something similar to files from gmakegen.py 952 953 Test depends on script which also depends on source 954 file, but since I don't have a good way generating 955 acting on a single file (oops) just depend on 956 executable which in turn will depend on src file 957 """ 958 # Different options for how to set up the targets 959 compileExecsFirst=False 960 961 # Open file 962 fd = open(output, 'w') 963 964 # Write out the sources 965 gendeps = self.gen_gnumake(fd) 966 967 # Write out the tests and execname targets 968 fd.write("\n#Tests and executables\n") # Delimiter 969 970 for pkg in self.pkg_pkgs: 971 # These grab the ones that are built 972 for lang in LANGS: 973 testdeps=[] 974 for ftest in self.tests[pkg][lang]: 975 test=os.path.basename(ftest) 976 basedir=os.path.dirname(ftest) 977 testdeps.append(self.nameSpace(test,basedir)) 978 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 979 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 980 981 # test targets 982 for ftest in self.tests[pkg][lang]: 983 test=os.path.basename(ftest) 984 basedir=os.path.dirname(ftest) 985 testdir="${TESTDIR}/"+basedir+"/" 986 nmtest=self.nameSpace(test,basedir) 987 rundir=os.path.join(testdir,test) 988 script=test+".sh" 989 990 # Deps 991 exfile=self.tests[pkg][lang][ftest]['exfile'] 992 fullex=os.path.join(self.srcdir,exfile) 993 localexec=self.tests[pkg][lang][ftest]['exec'] 994 execname=os.path.join(testdir,localexec) 995 fullscript=os.path.join(testdir,script) 996 tmpfile=os.path.join(testdir,test,test+".tmp") 997 998 # *.counts depends on the script and either executable (will 999 # be run) or the example source file (SKIP or TODO) 1000 fd.write('%s.counts : %s %s' 1001 % (os.path.join('$(TESTDIR)/counts', nmtest), 1002 fullscript, 1003 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1004 ) 1005 if exfile in self.sources[pkg][lang]: 1006 for dep in self.sources[pkg][lang][exfile]: 1007 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1008 fd.write('\n') 1009 1010 # Now write the args: 1011 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1012 1013 fd.close() 1014 return 1015 1016def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1017 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1018 srcdir=None, testdir=None, check=False): 1019 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1020 testdir=os.path.normpath(testdir) 1021 if petsc_arch: 1022 petsc_arch=petsc_arch.rstrip(os.path.sep) 1023 if len(petsc_arch.split(os.path.sep))>1: 1024 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1025 output = os.path.join(testdir, 'testfiles') 1026 1027 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1028 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1029 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1030 testdir=testdir,check=check) 1031 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1032 pEx.write_gnumake(dataDict, output) 1033 1034if __name__ == '__main__': 1035 import optparse 1036 parser = optparse.OptionParser() 1037 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1038 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1039 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1040 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1041 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1042 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1043 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1044 help='Check whether output files are in output director') 1045 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1046 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1047 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1048 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1049 1050 opts, extra_args = parser.parse_args() 1051 if extra_args: 1052 import sys 1053 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1054 exit(1) 1055 if opts.testdir is None: 1056 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1057 1058 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1059 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1060 verbose=opts.verbose, 1061 single_ex=opts.single_executable, srcdir=opts.srcdir, 1062 testdir=opts.testdir, check=opts.check_output) 1063