1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 if self.inInstallDir: 103 # Case 2 discussed above 104 # set PETSC_ARCH to install directory to allow script to work in both 105 dirlist=thisscriptdir.split(os.path.sep) 106 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 107 self.arch_dir=installdir 108 if self.srcdir is None: 109 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 110 else: 111 if petsc_arch == '': 112 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 113 # Case 1 discussed above 114 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 115 if self.srcdir is None: 116 self.srcdir=os.path.join(self.petsc_dir,'src') 117 118 self.testroot_dir=os.path.abspath(testdir) 119 120 self.verbose=verbose 121 # Whether to write out a useful debugging 122 self.summarize=True if verbose else False 123 124 # For help in setting the requirements 125 self.precision_types="single double __float128 int32".split() 126 self.integer_types="int32 int64 long32 long64".split() 127 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 128 129 # Things that are not test 130 self.buildkeys=testparse.buildkeys 131 132 # Adding a dictionary for storing sources, objects, and tests 133 # to make building the dependency tree easier 134 self.sources={} 135 self.objects={} 136 self.tests={} 137 for pkg in self.pkg_pkgs: 138 self.sources[pkg]={} 139 self.objects[pkg]=[] 140 self.tests[pkg]={} 141 for lang in LANGS: 142 self.sources[pkg][lang]={} 143 self.sources[pkg][lang]['srcs']=[] 144 self.tests[pkg][lang]={} 145 146 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 147 148 self.indent=" " 149 if self.verbose: print('Finishing the constructor') 150 return 151 152 def srcrelpath(self,rdir): 153 """ 154 Get relative path to source directory 155 """ 156 return os.path.relpath(rdir,self.srcdir) 157 158 def getInInstallDir(self,thisscriptdir): 159 """ 160 When petsc is installed then this file in installed in: 161 <PREFIX>/share/petsc/examples/config/gmakegentest.py 162 otherwise the path is: 163 <PETSC_DIR>/config/gmakegentest.py 164 We use this difference to determine if we are in installdir 165 """ 166 dirlist=thisscriptdir.split(os.path.sep) 167 if len(dirlist)>4: 168 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 169 if lastfour==os.path.join('share','petsc','examples','config'): 170 return True 171 else: 172 return False 173 else: 174 return False 175 176 def getLanguage(self,srcfile): 177 """ 178 Based on the source, determine associated language as found in gmakegen.LANGS 179 Can we just return srcext[1:\] now? 180 """ 181 langReq=None 182 srcext=os.path.splitext(srcfile)[-1] 183 if srcext in ".F90".split(): langReq="F90" 184 if srcext in ".F".split(): langReq="F" 185 if srcext in ".cxx".split(): langReq="cxx" 186 if srcext in ".cpp".split(): langReq="cpp" 187 if srcext == ".cu": langReq="cu" 188 if srcext == ".c": langReq="c" 189 #if not langReq: print("ERROR: ", srcext, srcfile) 190 return langReq 191 192 def _getAltList(self,output_file,srcdir): 193 ''' Calculate AltList based on output file-- see 194 src/snes/tutorials/output/ex22*.out 195 ''' 196 altlist=[output_file] 197 basefile,ext = os.path.splitext(output_file) 198 for i in range(1,9): 199 altroot=basefile+"_alt" 200 if i > 1: altroot=altroot+"_"+str(i) 201 af=altroot+".out" 202 srcaf=os.path.join(srcdir,af) 203 fullaf=os.path.join(self.petsc_dir,srcaf) 204 if os.path.isfile(fullaf): altlist.append(srcaf) 205 206 return altlist 207 208 209 def _getLoopVars(self,inDict,testname, isSubtest=False): 210 """ 211 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 212 Return: 213 inDict['args']: -ksp_monitor 214 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 215 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 216 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 217 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 218 subst should be passed in instead of inDict 219 """ 220 loopVars={}; newargs=[] 221 lsuffix='+' 222 argregex = re.compile(' (?=-[a-zA-Z])') 223 from testparse import parseLoopArgs 224 for key in inDict: 225 if key in ('SKIP', 'regexes'): 226 continue 227 akey=('subargs' if key=='args' else key) # what to assign 228 if akey not in inDict: inDict[akey]='' 229 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 230 # Always generate a loop over nsize, even if there is only one value 231 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 232 keystr = str(inDict[key]) 233 varlist = [] 234 for varset in argregex.split(keystr): 235 if not varset.strip(): continue 236 if '{{' in varset: 237 keyvar,lvars,ftype=parseLoopArgs(varset) 238 if akey not in loopVars: loopVars[akey]={} 239 varlist.append(keyvar) 240 loopVars[akey][keyvar]=[keyvar,lvars] 241 if akey=='nsize': 242 if len(lvars.split()) > 1: 243 lsuffix += akey +'-${i' + keyvar + '}' 244 else: 245 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 246 lsuffix+=keyvar+'-${i' + keyvar + '}_' 247 else: 248 if key=='args': 249 newargs.append(varset.strip()) 250 if varlist: 251 loopVars[akey]['varlist']=varlist 252 253 # For subtests, args are always substituted in (not top level) 254 if isSubtest: 255 inDict['subargs'] += " "+" ".join(newargs) 256 inDict['args']='' 257 if 'label_suffix' in inDict: 258 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 259 else: 260 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 261 else: 262 if loopVars: 263 inDict['args'] = ' '.join(newargs) 264 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 265 return loopVars 266 267 def getArgLabel(self,testDict): 268 """ 269 In all of the arguments in the test dictionary, create a simple 270 string for searching within the makefile system. For simplicity in 271 search, remove "-", for strings, etc. 272 Also, concatenate the arg commands 273 For now, ignore nsize -- seems hard to search for anyway 274 """ 275 # Collect all of the args associated with a test 276 argStr=("" if 'args' not in testDict else testDict['args']) 277 if 'subtests' in testDict: 278 for stest in testDict["subtests"]: 279 sd=testDict[stest] 280 argStr=argStr+("" if 'args' not in sd else sd['args']) 281 282 # Now go through and cleanup 283 argStr=re.sub('{{(.*?)}}',"",argStr) 284 argStr=re.sub('-'," ",argStr) 285 for digit in string.digits: argStr=re.sub(digit," ",argStr) 286 argStr=re.sub("\.","",argStr) 287 argStr=re.sub(",","",argStr) 288 argStr=re.sub('\+',' ',argStr) 289 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 290 return argStr.strip() 291 292 def addToSources(self,exfile,rpath,srcDict): 293 """ 294 Put into data structure that allows easy generation of makefile 295 """ 296 pkg=rpath.split(os.path.sep)[0] 297 relpfile=os.path.join(rpath,exfile) 298 lang=self.getLanguage(exfile) 299 if not lang: return 300 if pkg not in self.sources: return 301 self.sources[pkg][lang]['srcs'].append(relpfile) 302 self.sources[pkg][lang][relpfile] = [] 303 if 'depends' in srcDict: 304 depSrcList=srcDict['depends'].split() 305 for depSrc in depSrcList: 306 depObj=os.path.splitext(depSrc)[0]+".o" 307 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 308 309 # In gmakefile, ${TESTDIR} var specifies the object compilation 310 testsdir=rpath+"/" 311 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 312 self.objects[pkg].append(objfile) 313 return 314 315 def addToTests(self,test,rpath,exfile,execname,testDict): 316 """ 317 Put into data structure that allows easy generation of makefile 318 Organized by languages to allow testing of languages 319 """ 320 pkg=rpath.split("/")[0] 321 nmtest=os.path.join(rpath,test) 322 lang=self.getLanguage(exfile) 323 if not lang: return 324 if pkg not in self.tests: return 325 self.tests[pkg][lang][nmtest]={} 326 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 327 self.tests[pkg][lang][nmtest]['exec']=execname 328 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 329 return 330 331 def getExecname(self,exfile,rpath): 332 """ 333 Generate bash script using template found next to this file. 334 This file is read in at constructor time to avoid file I/O 335 """ 336 if self.single_ex: 337 execname=rpath.split("/")[1]+"-ex" 338 else: 339 execname=os.path.splitext(exfile)[0] 340 return execname 341 342 def getSubstVars(self,testDict,rpath,testname): 343 """ 344 Create a dictionary with all of the variables that get substituted 345 into the template commands found in example_template.py 346 """ 347 subst={} 348 349 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 350 if 'nsize' not in testDict: testDict['nsize'] = '1' 351 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 352 for ak in testparse.acceptedkeys: 353 if ak=='test': continue 354 subst[ak]=(testDict[ak] if ak in testDict else '') 355 356 # Now do other variables 357 subst['execname']=testDict['execname'] 358 subst['error']='' 359 if 'filter' in testDict: 360 if testDict['filter'].startswith("Error:"): 361 subst['error']="Error" 362 subst['filter']=testDict['filter'].lstrip("Error:") 363 else: 364 subst['filter']=testDict['filter'] 365 366 # Others 367 subst['subargs']='' # Default. For variables override 368 subst['srcdir']=os.path.join(self.srcdir, rpath) 369 subst['label_suffix']='' 370 subst['comments']="\n#".join(subst['comments'].split("\n")) 371 if subst['comments']: subst['comments']="#"+subst['comments'] 372 subst['exec']="../"+subst['execname'] 373 subst['testroot']=self.testroot_dir 374 subst['testname']=testname 375 dp = self.conf.get('DATAFILESPATH','') 376 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 377 378 # This is used to label some matrices 379 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 380 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 381 382 #Conf vars 383 if self.petsc_arch.find('valgrind')>=0: 384 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 385 else: 386 subst['mpiexec']=self.conf['MPIEXEC'] 387 subst['pkg_name']=self.pkg_name 388 subst['pkg_dir']=self.pkg_dir 389 subst['pkg_arch']=self.petsc_arch 390 subst['CONFIG_DIR']=thisscriptdir 391 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 392 subst['diff']=self.conf['DIFF'] 393 subst['rm']=self.conf['RM'] 394 subst['grep']=self.conf['GREP'] 395 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 396 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 397 398 # Output file is special because of subtests override 399 defroot = testparse.getDefaultOutputFileRoot(testname) 400 if 'output_file' not in testDict: 401 subst['output_file']="output/"+defroot+".out" 402 subst['redirect_file']=defroot+".tmp" 403 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 404 405 # Add in the full path here. 406 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 407 408 subst['regexes']={} 409 for subkey in subst: 410 if subkey=='regexes': continue 411 if not isinstance(subst[subkey],str): continue 412 patt="@"+subkey.upper()+"@" 413 subst['regexes'][subkey]=re.compile(patt) 414 415 return subst 416 417 def _substVars(self,subst,origStr): 418 """ 419 Substitute variables 420 """ 421 Str=origStr 422 for subkey in subst: 423 if subkey=='regexes': continue 424 if not isinstance(subst[subkey],str): continue 425 if subkey.upper() not in Str: continue 426 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 427 return Str 428 429 def getCmds(self,subst,i, debug=False): 430 """ 431 Generate bash script using template found next to this file. 432 This file is read in at constructor time to avoid file I/O 433 """ 434 nindnt=i # the start and has to be consistent with below 435 cmdindnt=self.indent*nindnt 436 cmdLines="" 437 438 # MPI is the default -- but we have a few odd commands 439 if not subst['command']: 440 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 441 else: 442 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 443 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 444 445 cmdLines+=cmdindnt+'if test $res = 0; then\n' 446 diffindnt=self.indent*(nindnt+1) 447 448 # Do some checks on existence of output_file and alt files 449 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 450 if not subst['TODO']: 451 print("Warning: "+subst['output_file']+" not found.") 452 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 453 454 # altlist always has output_file 455 if len(altlist)==1: 456 cmd=diffindnt+self._substVars(subst,example_template.difftest) 457 else: 458 if debug: print("Found alt files: ",altlist) 459 # Have to do it by hand a bit because of variable number of alt files 460 rf=subst['redirect_file'] 461 cmd=diffindnt+example_template.difftest.split('@')[0] 462 for i in range(len(altlist)): 463 af=altlist[i] 464 cmd+=af+' '+rf 465 if i!=len(altlist)-1: 466 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 467 cmd+=' || ${diff_exe} ' 468 else: 469 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 470 cmd+=subst['label_suffix']+' ""' # Quotes are painful 471 cmdLines+=cmd+"\n" 472 cmdLines+=cmdindnt+'else\n' 473 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 474 cmdLines+=cmdindnt+'fi\n' 475 return cmdLines 476 477 def _writeTodoSkip(self,fh,tors,reasons,footer): 478 """ 479 Write out the TODO and SKIP lines in the file 480 The TODO or SKIP variable, tors, should be lower case 481 """ 482 TORS=tors.upper() 483 template=eval("example_template."+tors+"line") 484 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 485 tab = '' 486 if reasons: 487 fh.write('if ! $force; then\n') 488 tab = tab + ' ' 489 if reasons == ["Requires DATAFILESPATH"]: 490 # The only reason not to run is DATAFILESPATH, which we check at run-time 491 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 492 tab = tab + ' ' 493 if reasons: 494 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 495 fh.write(tab+footer+"\n") 496 fh.write(tab+"exit\n") 497 if reasons == ["Requires DATAFILESPATH"]: 498 fh.write(' fi\n') 499 if reasons: 500 fh.write('fi\n') 501 fh.write('\n\n') 502 return 503 504 def getLoopVarsHead(self,loopVars,i,usedVars={}): 505 """ 506 Generate a nicely indented string with the format loops 507 Here is what the data structure looks like 508 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 509 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 510 loopVars['subargs']['pc_type']=["j","cholesky sor"] 511 """ 512 outstr=''; indnt=self.indent 513 514 for key in loopVars: 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 518 outstr += "\n\n" 519 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 varval=loopVars[key][var] 523 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 524 i = i + 1 525 return (outstr,i) 526 527 def getLoopVarsFoot(self,loopVars,i): 528 outstr=''; indnt=self.indent 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 i = i - 1 532 outstr += indnt * i + "done\n" 533 return (outstr,i) 534 535 def genRunScript(self,testname,root,isRun,srcDict): 536 """ 537 Generate bash script using template found next to this file. 538 This file is read in at constructor time to avoid file I/O 539 """ 540 # runscript_dir directory has to be consistent with gmakefile 541 testDict=srcDict[testname] 542 rpath=self.srcrelpath(root) 543 runscript_dir=os.path.join(self.testroot_dir,rpath) 544 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 545 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 546 547 # Get variables to go into shell scripts. last time testDict used 548 subst=self.getSubstVars(testDict,rpath,testname) 549 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 550 if 'subtests' in testDict: 551 # The subtests inherit inDict, so we don't need top-level loops. 552 loopVars = {} 553 554 #Handle runfiles 555 for lfile in subst.get('localrunfiles','').split(): 556 install_files(os.path.join(root, lfile), 557 os.path.join(runscript_dir, os.path.dirname(lfile))) 558 # Check subtests for local runfiles 559 for stest in subst.get("subtests",[]): 560 for lfile in testDict[stest].get('localrunfiles','').split(): 561 install_files(os.path.join(root, lfile), 562 os.path.join(runscript_dir, os.path.dirname(lfile))) 563 564 # Now substitute the key variables into the header and footer 565 header=self._substVars(subst,example_template.header) 566 # The header is done twice to enable @...@ in header 567 header=self._substVars(subst,header) 568 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 569 570 # Start writing the file 571 fh.write(header+"\n") 572 573 # If there is a TODO or a SKIP then we do it before writing out the 574 # rest of the command (which is useful for working on the test) 575 # SKIP and TODO can be for the source file or for the runs 576 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 577 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 578 579 j=0 # for indentation 580 581 if loopVars: 582 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 583 if (loopHead): fh.write(loopHead+"\n") 584 585 # Subtests are special 586 allLoopVars=list(loopVars.keys()) 587 if 'subtests' in testDict: 588 substP=subst # Subtests can inherit args but be careful 589 k=0 # for label suffixes 590 for stest in testDict["subtests"]: 591 subst=substP.copy() 592 subst.update(testDict[stest]) 593 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 594 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 595 if sLoopVars: 596 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 597 allLoopVars+=list(sLoopVars.keys()) 598 fh.write(sLoopHead+"\n") 599 fh.write(self.getCmds(subst,j)+"\n") 600 if sLoopVars: 601 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 602 fh.write(sLoopFoot+"\n") 603 else: 604 fh.write(self.getCmds(subst,j)+"\n") 605 606 if loopVars: 607 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 608 fh.write(loopFoot+"\n") 609 610 fh.write(footer+"\n") 611 612 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 613 #if '10_9' in testname: sys.exit() 614 return 615 616 def genScriptsAndInfo(self,exfile,root,srcDict): 617 """ 618 Generate scripts from the source file, determine if built, etc. 619 For every test in the exfile with info in the srcDict: 620 1. Determine if it needs to be run for this arch 621 2. Generate the script 622 3. Generate the data needed to write out the makefile in a 623 convenient way 624 All tests are *always* run, but some may be SKIP'd per the TAP standard 625 """ 626 debug=False 627 rpath=self.srcrelpath(root) 628 execname=self.getExecname(exfile,rpath) 629 isBuilt=self._isBuilt(exfile,srcDict) 630 for test in srcDict: 631 if test in self.buildkeys: continue 632 if debug: print(nameSpace(exfile,root), test) 633 srcDict[test]['execname']=execname # Convenience in generating scripts 634 isRun=self._isRun(srcDict[test]) 635 self.genRunScript(test,root,isRun,srcDict) 636 srcDict[test]['isrun']=isRun 637 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 638 639 # This adds to datastructure for building deps 640 if isBuilt: self.addToSources(exfile,rpath,srcDict) 641 return 642 643 def _isBuilt(self,exfile,srcDict): 644 """ 645 Determine if this file should be built. 646 """ 647 # Get the language based on file extension 648 srcDict['SKIP'] = [] 649 lang=self.getLanguage(exfile) 650 if (lang=="F" or lang=="F90"): 651 if not self.have_fortran: 652 srcDict["SKIP"].append("Fortran required for this test") 653 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 654 srcDict["SKIP"].append("Fortran f90freeform required for this test") 655 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 656 srcDict["SKIP"].append("CUDA required for this test") 657 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 658 srcDict["SKIP"].append("C++ required for this test") 659 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 660 srcDict["SKIP"].append("C++ required for this test") 661 662 # Deprecated source files 663 if srcDict.get("TODO"): 664 return False 665 666 # isRun can work with srcDict to handle the requires 667 if "requires" in srcDict: 668 if srcDict["requires"]: 669 return self._isRun(srcDict) 670 671 return srcDict['SKIP'] == [] 672 673 674 def _isRun(self,testDict, debug=False): 675 """ 676 Based on the requirements listed in the src file and the petscconf.h 677 info, determine whether this test should be run or not. 678 """ 679 indent=" " 680 681 if 'SKIP' not in testDict: 682 testDict['SKIP'] = [] 683 # MPI requirements 684 if 'MPI_IS_MPIUNI' in self.conf: 685 if testDict.get('nsize', '1') != '1': 686 testDict['SKIP'].append("Parallel test with serial build") 687 688 # The requirements for the test are the sum of all the run subtests 689 if 'subtests' in testDict: 690 if 'requires' not in testDict: testDict['requires']="" 691 for stest in testDict['subtests']: 692 if 'requires' in testDict[stest]: 693 testDict['requires']+=" "+testDict[stest]['requires'] 694 if testDict[stest].get('nsize', '1') != '1': 695 testDict['SKIP'].append("Parallel test with serial build") 696 break 697 698 # Now go through all requirements 699 if 'requires' in testDict: 700 for requirement in testDict['requires'].split(): 701 requirement=requirement.strip() 702 if not requirement: continue 703 if debug: print(indent+"Requirement: ", requirement) 704 isNull=False 705 if requirement.startswith("!"): 706 requirement=requirement[1:]; isNull=True 707 # Precision requirement for reals 708 if requirement in self.precision_types: 709 if self.conf['PETSC_PRECISION']==requirement: 710 if isNull: 711 testDict['SKIP'].append("not "+requirement+" required") 712 continue 713 continue # Success 714 elif not isNull: 715 testDict['SKIP'].append(requirement+" required") 716 continue 717 # Precision requirement for ints 718 if requirement in self.integer_types: 719 if requirement=="int32": 720 if self.conf['PETSC_SIZEOF_INT']==4: 721 if isNull: 722 testDict['SKIP'].append("not int32 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int32 required") 727 continue 728 if requirement=="int64": 729 if self.conf['PETSC_SIZEOF_INT']==8: 730 if isNull: 731 testDict['SKIP'].append("NOT int64 required") 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append("int64 required") 736 continue 737 if requirement.startswith("long"): 738 reqsize = int(requirement[4:])//8 739 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 740 if longsize==reqsize: 741 if isNull: 742 testDict['SKIP'].append("not %s required" % requirement) 743 continue 744 continue # Success 745 elif not isNull: 746 testDict['SKIP'].append("%s required" % requirement) 747 continue 748 # Datafilespath 749 if requirement=="datafilespath" and not isNull: 750 testDict['SKIP'].append("Requires DATAFILESPATH") 751 continue 752 # Defines -- not sure I have comments matching 753 if "define(" in requirement.lower(): 754 reqdef=requirement.split("(")[1].split(")")[0] 755 if reqdef in self.conf: 756 if isNull: 757 testDict['SKIP'].append("Null requirement not met: "+requirement) 758 continue 759 continue # Success 760 elif not isNull: 761 testDict['SKIP'].append("Required: "+requirement) 762 continue 763 764 # Rest should be packages that we can just get from conf 765 if requirement in ["complex","debug"]: 766 petscconfvar="PETSC_USE_"+requirement.upper() 767 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 768 else: 769 petscconfvar="PETSC_HAVE_"+requirement.upper() 770 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 771 petsccv = self.conf.get(petscconfvar) 772 pkgcv = self.conf.get(pkgconfvar) 773 774 if petsccv or pkgcv: 775 if isNull: 776 if petsccv: 777 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 778 continue 779 else: 780 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 781 continue 782 continue # Success 783 elif not isNull: 784 if not petsccv and not pkgcv: 785 if debug: print("requirement not found: ", requirement) 786 if self.pkg_name == 'petsc': 787 testDict['SKIP'].append(petscconfvar+" requirement not met") 788 else: 789 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 790 continue 791 return testDict['SKIP'] == [] 792 793 def checkOutput(self,exfile,root,srcDict): 794 """ 795 Check and make sure the output files are in the output director 796 """ 797 debug=False 798 rpath=self.srcrelpath(root) 799 for test in srcDict: 800 if test in self.buildkeys: continue 801 if debug: print(rpath, exfile, test) 802 if 'output_file' in srcDict[test]: 803 output_file=srcDict[test]['output_file'] 804 else: 805 defroot = testparse.getDefaultOutputFileRoot(test) 806 if 'TODO' in srcDict[test]: continue 807 output_file="output/"+defroot+".out" 808 809 fullout=os.path.join(root,output_file) 810 if debug: print("---> ",fullout) 811 if not os.path.exists(fullout): 812 self.missing_files.append(fullout) 813 814 return 815 816 def genPetscTests_summarize(self,dataDict): 817 """ 818 Required method to state what happened 819 """ 820 if not self.summarize: return 821 indent=" " 822 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 823 with open(fhname, "w") as fh: 824 for root in dataDict: 825 relroot=self.srcrelpath(root) 826 pkg=relroot.split("/")[1] 827 fh.write(relroot+"\n") 828 allSrcs=[] 829 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 830 for exfile in dataDict[root]: 831 # Basic information 832 rfile=os.path.join(relroot,exfile) 833 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 834 fh.write(indent+exfile+indent*4+builtStatus+"\n") 835 for test in dataDict[root][exfile]: 836 if test in self.buildkeys: continue 837 line=indent*2+test 838 fh.write(line+"\n") 839 # Looks nice to have the keys in order 840 #for key in dataDict[root][exfile][test]: 841 for key in "isrun abstracted nsize args requires script".split(): 842 if key not in dataDict[root][exfile][test]: continue 843 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 844 fh.write(line+"\n") 845 fh.write("\n") 846 fh.write("\n") 847 fh.write("\n") 848 return 849 850 def genPetscTests(self,root,dirs,files,dataDict): 851 """ 852 Go through and parse the source files in the directory to generate 853 the examples based on the metadata contained in the source files 854 """ 855 debug=False 856 # Use examplesAnalyze to get what the makefles think are sources 857 #self.examplesAnalyze(root,dirs,files,anlzDict) 858 859 dataDict[root]={} 860 861 for exfile in files: 862 #TST: Until we replace files, still leaving the orginals as is 863 #if not exfile.startswith("new_"+"ex"): continue 864 #if not exfile.startswith("ex"): continue 865 866 # Ignore emacs and other temporary files 867 if exfile.startswith("."): continue 868 if exfile.startswith("#"): continue 869 if exfile.endswith("~"): continue 870 # Only parse source files 871 ext=os.path.splitext(exfile)[-1].lstrip('.') 872 if ext not in LANGS: continue 873 874 # Convenience 875 fullex=os.path.join(root,exfile) 876 if self.verbose: print(' --> '+fullex) 877 dataDict[root].update(testparse.parseTestFile(fullex,0)) 878 if exfile in dataDict[root]: 879 if not self.check_output: 880 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 881 else: 882 self.checkOutput(exfile,root,dataDict[root][exfile]) 883 884 return 885 886 def walktree(self,top): 887 """ 888 Walk a directory tree, starting from 'top' 889 """ 890 if self.check_output: 891 print("Checking for missing output files") 892 self.missing_files=[] 893 894 # Goal of action is to fill this dictionary 895 dataDict={} 896 for root, dirs, files in os.walk(top, topdown=True): 897 dirs.sort() 898 files.sort() 899 if "/tests" not in root and "/tutorials" not in root: continue 900 if "dSYM" in root: continue 901 if os.path.basename(root.rstrip("/")) == 'output': continue 902 if self.verbose: print(root) 903 self.genPetscTests(root,dirs,files,dataDict) 904 905 # If checking output, report results 906 if self.check_output: 907 if self.missing_files: 908 for file in set(self.missing_files): # set uniqifies 909 print(file) 910 sys.exit(1) 911 912 # Now summarize this dictionary 913 if self.verbose: self.genPetscTests_summarize(dataDict) 914 return dataDict 915 916 def gen_gnumake(self, fd): 917 """ 918 Overwrite of the method in the base PETSc class 919 """ 920 def write(stem, srcs): 921 for lang in LANGS: 922 if srcs[lang]['srcs']: 923 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 924 for pkg in self.pkg_pkgs: 925 srcs = self.gen_pkg(pkg) 926 write('testsrcs-' + pkg, srcs) 927 # Handle dependencies 928 for lang in LANGS: 929 for exfile in srcs[lang]['srcs']: 930 if exfile in srcs[lang]: 931 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 932 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 933 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 934 if deps: 935 # The executable literally depends on the object file because it is linked 936 fd.write(ex +": " + " ".join(deps) +'\n') 937 # The object file containing 'main' does not normally depend on other object 938 # files, but it does when it includes their modules. This dependency is 939 # overly blunt and could be reduced to only depend on object files for 940 # modules that are used, like "*f90aux.o". 941 fd.write(exfo +": " + " ".join(deps) +'\n') 942 943 return self.gendeps 944 945 def gen_pkg(self, pkg): 946 """ 947 Overwrite of the method in the base PETSc class 948 """ 949 return self.sources[pkg] 950 951 def write_gnumake(self, dataDict, output=None): 952 """ 953 Write out something similar to files from gmakegen.py 954 955 Test depends on script which also depends on source 956 file, but since I don't have a good way generating 957 acting on a single file (oops) just depend on 958 executable which in turn will depend on src file 959 """ 960 # Different options for how to set up the targets 961 compileExecsFirst=False 962 963 # Open file 964 with open(output, 'w') as fd: 965 # Write out the sources 966 gendeps = self.gen_gnumake(fd) 967 968 # Write out the tests and execname targets 969 fd.write("\n#Tests and executables\n") # Delimiter 970 971 for pkg in self.pkg_pkgs: 972 # These grab the ones that are built 973 for lang in LANGS: 974 testdeps=[] 975 for ftest in self.tests[pkg][lang]: 976 test=os.path.basename(ftest) 977 basedir=os.path.dirname(ftest) 978 testdeps.append(nameSpace(test,basedir)) 979 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 980 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 981 982 # test targets 983 for ftest in self.tests[pkg][lang]: 984 test=os.path.basename(ftest) 985 basedir=os.path.dirname(ftest) 986 testdir="${TESTDIR}/"+basedir+"/" 987 nmtest=nameSpace(test,basedir) 988 rundir=os.path.join(testdir,test) 989 script=test+".sh" 990 991 # Deps 992 exfile=self.tests[pkg][lang][ftest]['exfile'] 993 fullex=os.path.join(self.srcdir,exfile) 994 localexec=self.tests[pkg][lang][ftest]['exec'] 995 execname=os.path.join(testdir,localexec) 996 fullscript=os.path.join(testdir,script) 997 tmpfile=os.path.join(testdir,test,test+".tmp") 998 999 # *.counts depends on the script and either executable (will 1000 # be run) or the example source file (SKIP or TODO) 1001 fd.write('%s.counts : %s %s' 1002 % (os.path.join('$(TESTDIR)/counts', nmtest), 1003 fullscript, 1004 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1005 ) 1006 if exfile in self.sources[pkg][lang]: 1007 for dep in self.sources[pkg][lang][exfile]: 1008 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1009 fd.write('\n') 1010 1011 # Now write the args: 1012 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1013 1014 return 1015 1016 def write_db(self, dataDict, testdir): 1017 """ 1018 Write out the dataDict into a pickle file 1019 """ 1020 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1021 pickle.dump(dataDict,fd) 1022 return 1023 1024def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1025 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1026 srcdir=None, testdir=None, check=False): 1027 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1028 testdir=os.path.normpath(testdir) 1029 if petsc_arch: 1030 petsc_arch=petsc_arch.rstrip(os.path.sep) 1031 if len(petsc_arch.split(os.path.sep))>1: 1032 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1033 output = os.path.join(testdir, 'testfiles') 1034 1035 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1036 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1037 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1038 testdir=testdir,check=check) 1039 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1040 if not pEx.check_output: 1041 pEx.write_gnumake(dataDict, output) 1042 pEx.write_db(dataDict, testdir) 1043 1044if __name__ == '__main__': 1045 import optparse 1046 parser = optparse.OptionParser() 1047 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1048 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1049 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1050 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1051 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1052 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1053 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1054 help='Check whether output files are in output director') 1055 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1056 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1057 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1058 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1059 1060 opts, extra_args = parser.parse_args() 1061 if extra_args: 1062 import sys 1063 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1064 exit(1) 1065 if opts.testdir is None: 1066 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1067 1068 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1069 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1070 verbose=opts.verbose, 1071 single_ex=opts.single_executable, srcdir=opts.srcdir, 1072 testdir=opts.testdir, check=opts.check_output) 1073