1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace('/examples/','_').replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 if self.inInstallDir: 103 # Case 2 discussed above 104 # set PETSC_ARCH to install directory to allow script to work in both 105 dirlist=thisscriptdir.split(os.path.sep) 106 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 107 self.arch_dir=installdir 108 if self.srcdir is None: 109 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 110 else: 111 if petsc_arch == '': 112 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 113 # Case 1 discussed above 114 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 115 if self.srcdir is None: 116 self.srcdir=os.path.join(self.petsc_dir,'src') 117 118 self.testroot_dir=os.path.abspath(testdir) 119 120 self.verbose=verbose 121 # Whether to write out a useful debugging 122 self.summarize=True if verbose else False 123 124 # For help in setting the requirements 125 self.precision_types="single double __float128 int32".split() 126 self.integer_types="int32 int64 long32 long64".split() 127 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 128 129 # Things that are not test 130 self.buildkeys=testparse.buildkeys 131 132 # Adding a dictionary for storing sources, objects, and tests 133 # to make building the dependency tree easier 134 self.sources={} 135 self.objects={} 136 self.tests={} 137 for pkg in self.pkg_pkgs: 138 self.sources[pkg]={} 139 self.objects[pkg]=[] 140 self.tests[pkg]={} 141 for lang in LANGS: 142 self.sources[pkg][lang]={} 143 self.sources[pkg][lang]['srcs']=[] 144 self.tests[pkg][lang]={} 145 146 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 147 148 self.indent=" " 149 if self.verbose: print('Finishing the constructor') 150 return 151 152 def srcrelpath(self,rdir): 153 """ 154 Get relative path to source directory 155 """ 156 return os.path.relpath(rdir,self.srcdir) 157 158 def getInInstallDir(self,thisscriptdir): 159 """ 160 When petsc is installed then this file in installed in: 161 <PREFIX>/share/petsc/examples/config/gmakegentest.py 162 otherwise the path is: 163 <PETSC_DIR>/config/gmakegentest.py 164 We use this difference to determine if we are in installdir 165 """ 166 dirlist=thisscriptdir.split(os.path.sep) 167 if len(dirlist)>4: 168 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 169 if lastfour==os.path.join('share','petsc','examples','config'): 170 return True 171 else: 172 return False 173 else: 174 return False 175 176 def getLanguage(self,srcfile): 177 """ 178 Based on the source, determine associated language as found in gmakegen.LANGS 179 Can we just return srcext[1:\] now? 180 """ 181 langReq=None 182 srcext=os.path.splitext(srcfile)[-1] 183 if srcext in ".F90".split(): langReq="F90" 184 if srcext in ".F".split(): langReq="F" 185 if srcext in ".cxx".split(): langReq="cxx" 186 if srcext in ".cpp".split(): langReq="cpp" 187 if srcext == ".cu": langReq="cu" 188 if srcext == ".c": langReq="c" 189 #if not langReq: print("ERROR: ", srcext, srcfile) 190 return langReq 191 192 def _getLoopVars(self,inDict,testname, isSubtest=False): 193 """ 194 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 195 Return: 196 inDict['args']: -ksp_monitor 197 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 198 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 199 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 200 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 201 subst should be passed in instead of inDict 202 """ 203 loopVars={}; newargs=[] 204 lsuffix='+' 205 argregex = re.compile(' (?=-[a-zA-Z])') 206 from testparse import parseLoopArgs 207 for key in inDict: 208 if key in ('SKIP', 'regexes'): 209 continue 210 akey=('subargs' if key=='args' else key) # what to assign 211 if akey not in inDict: inDict[akey]='' 212 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 213 # Always generate a loop over nsize, even if there is only one value 214 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 215 keystr = str(inDict[key]) 216 varlist = [] 217 for varset in argregex.split(keystr): 218 if not varset.strip(): continue 219 if '{{' in varset: 220 keyvar,lvars,ftype=parseLoopArgs(varset) 221 if akey not in loopVars: loopVars[akey]={} 222 varlist.append(keyvar) 223 loopVars[akey][keyvar]=[keyvar,lvars] 224 if akey=='nsize': 225 if len(lvars.split()) > 1: 226 lsuffix += akey +'-${' + keyvar + '}' 227 else: 228 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 229 lsuffix+=keyvar+'-${' + keyvar + '}_' 230 else: 231 if key=='args': 232 newargs.append(varset.strip()) 233 if varlist: 234 loopVars[akey]['varlist']=varlist 235 236 # For subtests, args are always substituted in (not top level) 237 if isSubtest: 238 inDict['subargs'] += " "+" ".join(newargs) 239 inDict['args']='' 240 if 'label_suffix' in inDict: 241 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 242 else: 243 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 244 else: 245 if loopVars: 246 inDict['args'] = ' '.join(newargs) 247 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 248 return loopVars 249 250 def getArgLabel(self,testDict): 251 """ 252 In all of the arguments in the test dictionary, create a simple 253 string for searching within the makefile system. For simplicity in 254 search, remove "-", for strings, etc. 255 Also, concatenate the arg commands 256 For now, ignore nsize -- seems hard to search for anyway 257 """ 258 # Collect all of the args associated with a test 259 argStr=("" if 'args' not in testDict else testDict['args']) 260 if 'subtests' in testDict: 261 for stest in testDict["subtests"]: 262 sd=testDict[stest] 263 argStr=argStr+("" if 'args' not in sd else sd['args']) 264 265 # Now go through and cleanup 266 argStr=re.sub('{{(.*?)}}',"",argStr) 267 argStr=re.sub('-'," ",argStr) 268 for digit in string.digits: argStr=re.sub(digit," ",argStr) 269 argStr=re.sub("\.","",argStr) 270 argStr=re.sub(",","",argStr) 271 argStr=re.sub('\+',' ',argStr) 272 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 273 return argStr.strip() 274 275 def addToSources(self,exfile,rpath,srcDict): 276 """ 277 Put into data structure that allows easy generation of makefile 278 """ 279 pkg=rpath.split(os.path.sep)[0] 280 relpfile=os.path.join(rpath,exfile) 281 lang=self.getLanguage(exfile) 282 if not lang: return 283 if pkg not in self.sources: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=rpath+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,rpath,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 pkg=rpath.split("/")[0] 304 nmtest=os.path.join(rpath,test) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 if pkg not in self.tests: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,rpath): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 if self.single_ex: 320 execname=rpath.split("/")[1]+"-ex" 321 else: 322 execname=os.path.splitext(exfile)[0] 323 return execname 324 325 def getSubstVars(self,testDict,rpath,testname): 326 """ 327 Create a dictionary with all of the variables that get substituted 328 into the template commands found in example_template.py 329 """ 330 subst={} 331 332 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 333 if 'nsize' not in testDict: testDict['nsize'] = '1' 334 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 335 for ak in testparse.acceptedkeys: 336 if ak=='test': continue 337 subst[ak]=(testDict[ak] if ak in testDict else '') 338 339 # Now do other variables 340 subst['execname']=testDict['execname'] 341 subst['error']='' 342 if 'filter' in testDict: 343 if testDict['filter'].startswith("Error:"): 344 subst['error']="Error" 345 subst['filter']=testDict['filter'].lstrip("Error:") 346 else: 347 subst['filter']=testDict['filter'] 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(self.srcdir, rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['pkg_name']=self.pkg_name 371 subst['pkg_dir']=self.pkg_dir 372 subst['pkg_arch']=self.petsc_arch 373 subst['CONFIG_DIR']=thisscriptdir 374 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 375 subst['diff']=self.conf['DIFF'] 376 subst['rm']=self.conf['RM'] 377 subst['grep']=self.conf['GREP'] 378 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 379 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 380 381 # Output file is special because of subtests override 382 defroot = testparse.getDefaultOutputFileRoot(testname) 383 if 'output_file' not in testDict: 384 subst['output_file']="output/"+defroot+".out" 385 subst['redirect_file']=defroot+".tmp" 386 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 387 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/examples/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if 'altfiles' not in subst: 447 cmd=diffindnt+self._substVars(subst,example_template.difftest) 448 else: 449 # Have to do it by hand a bit because of variable number of alt files 450 rf=subst['redirect_file'] 451 cmd=diffindnt+example_template.difftest.split('@')[0] 452 for i in range(len(subst['altfiles'])): 453 af=subst['altfiles'][i] 454 cmd+=af+' '+rf 455 if i!=len(subst['altfiles'])-1: 456 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 457 cmd+=' || ${diff_exe} ' 458 else: 459 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 460 cmd+=subst['label_suffix']+' ""' # Quotes are painful 461 cmdLines+=cmd+"\n" 462 cmdLines+=cmdindnt+'else\n' 463 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 464 cmdLines+=cmdindnt+'fi\n' 465 return cmdLines 466 467 def _writeTodoSkip(self,fh,tors,reasons,footer): 468 """ 469 Write out the TODO and SKIP lines in the file 470 The TODO or SKIP variable, tors, should be lower case 471 """ 472 TORS=tors.upper() 473 template=eval("example_template."+tors+"line") 474 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 475 tab = '' 476 if reasons: 477 fh.write('if ! $force; then\n') 478 tab = tab + ' ' 479 if reasons == ["Requires DATAFILESPATH"]: 480 # The only reason not to run is DATAFILESPATH, which we check at run-time 481 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 482 tab = tab + ' ' 483 if reasons: 484 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 485 fh.write(tab+footer+"\n") 486 fh.write(tab+"exit\n") 487 if reasons == ["Requires DATAFILESPATH"]: 488 fh.write(' fi\n') 489 if reasons: 490 fh.write('fi\n') 491 fh.write('\n\n') 492 return 493 494 def getLoopVarsHead(self,loopVars,i,usedVars={}): 495 """ 496 Generate a nicely indented string with the format loops 497 Here is what the data structure looks like 498 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 499 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 500 loopVars['subargs']['pc_type']=["j","cholesky sor"] 501 """ 502 outstr=''; indnt=self.indent 503 504 for key in loopVars: 505 if key in usedVars: continue # Do not duplicate setting vars 506 for var in loopVars[key]['varlist']: 507 varval=loopVars[key][var] 508 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 509 outstr += "\n\n" 510 511 for key in loopVars: 512 for var in loopVars[key]['varlist']: 513 varval=loopVars[key][var] 514 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 515 i = i + 1 516 return (outstr,i) 517 518 def getLoopVarsFoot(self,loopVars,i): 519 outstr=''; indnt=self.indent 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 i = i - 1 523 outstr += indnt * i + "done\n" 524 return (outstr,i) 525 526 def genRunScript(self,testname,root,isRun,srcDict): 527 """ 528 Generate bash script using template found next to this file. 529 This file is read in at constructor time to avoid file I/O 530 """ 531 # runscript_dir directory has to be consistent with gmakefile 532 testDict=srcDict[testname] 533 rpath=self.srcrelpath(root) 534 runscript_dir=os.path.join(self.testroot_dir,rpath) 535 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 536 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 537 538 # Get variables to go into shell scripts. last time testDict used 539 subst=self.getSubstVars(testDict,rpath,testname) 540 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 541 if 'subtests' in testDict: 542 # The subtests inherit inDict, so we don't need top-level loops. 543 loopVars = {} 544 545 #Handle runfiles 546 for lfile in subst.get('localrunfiles','').split(): 547 install_files(os.path.join(root, lfile), 548 os.path.join(runscript_dir, os.path.dirname(lfile))) 549 # Check subtests for local runfiles 550 for stest in subst.get("subtests",[]): 551 for lfile in testDict[stest].get('localrunfiles','').split(): 552 install_files(os.path.join(root, lfile), 553 os.path.join(runscript_dir, os.path.dirname(lfile))) 554 555 # Now substitute the key variables into the header and footer 556 header=self._substVars(subst,example_template.header) 557 # The header is done twice to enable @...@ in header 558 header=self._substVars(subst,header) 559 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 560 561 # Start writing the file 562 fh.write(header+"\n") 563 564 # If there is a TODO or a SKIP then we do it before writing out the 565 # rest of the command (which is useful for working on the test) 566 # SKIP and TODO can be for the source file or for the runs 567 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 568 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 569 570 j=0 # for indentation 571 572 if loopVars: 573 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 574 if (loopHead): fh.write(loopHead+"\n") 575 576 # Subtests are special 577 allLoopVars=list(loopVars.keys()) 578 if 'subtests' in testDict: 579 substP=subst # Subtests can inherit args but be careful 580 k=0 # for label suffixes 581 for stest in testDict["subtests"]: 582 subst=substP.copy() 583 subst.update(testDict[stest]) 584 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 585 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 586 if sLoopVars: 587 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 588 allLoopVars+=list(sLoopVars.keys()) 589 fh.write(sLoopHead+"\n") 590 fh.write(self.getCmds(subst,j)+"\n") 591 if sLoopVars: 592 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 593 fh.write(sLoopFoot+"\n") 594 else: 595 fh.write(self.getCmds(subst,j)+"\n") 596 597 if loopVars: 598 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 599 fh.write(loopFoot+"\n") 600 601 fh.write(footer+"\n") 602 fh.close() 603 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 604 #if '10_9' in testname: sys.exit() 605 return 606 607 def genScriptsAndInfo(self,exfile,root,srcDict): 608 """ 609 Generate scripts from the source file, determine if built, etc. 610 For every test in the exfile with info in the srcDict: 611 1. Determine if it needs to be run for this arch 612 2. Generate the script 613 3. Generate the data needed to write out the makefile in a 614 convenient way 615 All tests are *always* run, but some may be SKIP'd per the TAP standard 616 """ 617 debug=False 618 rpath=self.srcrelpath(root) 619 execname=self.getExecname(exfile,rpath) 620 isBuilt=self._isBuilt(exfile,srcDict) 621 for test in srcDict: 622 if test in self.buildkeys: continue 623 if debug: print(nameSpace(exfile,root), test) 624 srcDict[test]['execname']=execname # Convenience in generating scripts 625 isRun=self._isRun(srcDict[test]) 626 self.genRunScript(test,root,isRun,srcDict) 627 srcDict[test]['isrun']=isRun 628 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 629 630 # This adds to datastructure for building deps 631 if isBuilt: self.addToSources(exfile,rpath,srcDict) 632 return 633 634 def _isBuilt(self,exfile,srcDict): 635 """ 636 Determine if this file should be built. 637 """ 638 # Get the language based on file extension 639 srcDict['SKIP'] = [] 640 lang=self.getLanguage(exfile) 641 if (lang=="F" or lang=="F90"): 642 if not self.have_fortran: 643 srcDict["SKIP"].append("Fortran required for this test") 644 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 645 srcDict["SKIP"].append("Fortran f90freeform required for this test") 646 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 647 srcDict["SKIP"].append("CUDA required for this test") 648 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 649 srcDict["SKIP"].append("C++ required for this test") 650 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 651 srcDict["SKIP"].append("C++ required for this test") 652 653 # Deprecated source files 654 if srcDict.get("TODO"): 655 return False 656 657 # isRun can work with srcDict to handle the requires 658 if "requires" in srcDict: 659 if srcDict["requires"]: 660 return self._isRun(srcDict) 661 662 return srcDict['SKIP'] == [] 663 664 665 def _isRun(self,testDict, debug=False): 666 """ 667 Based on the requirements listed in the src file and the petscconf.h 668 info, determine whether this test should be run or not. 669 """ 670 indent=" " 671 672 if 'SKIP' not in testDict: 673 testDict['SKIP'] = [] 674 # MPI requirements 675 if 'MPI_IS_MPIUNI' in self.conf: 676 if testDict.get('nsize', '1') != '1': 677 testDict['SKIP'].append("Parallel test with serial build") 678 679 # The requirements for the test are the sum of all the run subtests 680 if 'subtests' in testDict: 681 if 'requires' not in testDict: testDict['requires']="" 682 for stest in testDict['subtests']: 683 if 'requires' in testDict[stest]: 684 testDict['requires']+=" "+testDict[stest]['requires'] 685 if testDict.get('nsize', '1') != '1': 686 testDict['SKIP'].append("Parallel test with serial build") 687 break 688 689 # Now go through all requirements 690 if 'requires' in testDict: 691 for requirement in testDict['requires'].split(): 692 requirement=requirement.strip() 693 if not requirement: continue 694 if debug: print(indent+"Requirement: ", requirement) 695 isNull=False 696 if requirement.startswith("!"): 697 requirement=requirement[1:]; isNull=True 698 # Precision requirement for reals 699 if requirement in self.precision_types: 700 if self.conf['PETSC_PRECISION']==requirement: 701 if isNull: 702 testDict['SKIP'].append("not "+requirement+" required") 703 continue 704 continue # Success 705 elif not isNull: 706 testDict['SKIP'].append(requirement+" required") 707 continue 708 # Precision requirement for ints 709 if requirement in self.integer_types: 710 if requirement=="int32": 711 if self.conf['PETSC_SIZEOF_INT']==4: 712 if isNull: 713 testDict['SKIP'].append("not int32 required") 714 continue 715 continue # Success 716 elif not isNull: 717 testDict['SKIP'].append("int32 required") 718 continue 719 if requirement=="int64": 720 if self.conf['PETSC_SIZEOF_INT']==8: 721 if isNull: 722 testDict['SKIP'].append("NOT int64 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int64 required") 727 continue 728 if requirement.startswith("long"): 729 reqsize = int(requirement[4:])//8 730 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 731 if longsize==reqsize: 732 if isNull: 733 testDict['SKIP'].append("not %s required" % requirement) 734 continue 735 continue # Success 736 elif not isNull: 737 testDict['SKIP'].append("%s required" % requirement) 738 continue 739 # Datafilespath 740 if requirement=="datafilespath" and not isNull: 741 testDict['SKIP'].append("Requires DATAFILESPATH") 742 continue 743 # Defines -- not sure I have comments matching 744 if "define(" in requirement.lower(): 745 reqdef=requirement.split("(")[1].split(")")[0] 746 if reqdef in self.conf: 747 if isNull: 748 testDict['SKIP'].append("Null requirement not met: "+requirement) 749 continue 750 continue # Success 751 elif not isNull: 752 testDict['SKIP'].append("Required: "+requirement) 753 continue 754 755 # Rest should be packages that we can just get from conf 756 if requirement in ["complex","debug"]: 757 petscconfvar="PETSC_USE_"+requirement.upper() 758 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 759 else: 760 petscconfvar="PETSC_HAVE_"+requirement.upper() 761 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 762 petsccv = self.conf.get(petscconfvar) 763 pkgcv = self.conf.get(pkgconfvar) 764 765 if petsccv or pkgcv: 766 if isNull: 767 if petsccv: 768 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 769 continue 770 else: 771 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 772 continue 773 continue # Success 774 elif not isNull: 775 if not petsccv and not pkgcv: 776 if debug: print("requirement not found: ", requirement) 777 if self.pkg_name == 'petsc': 778 testDict['SKIP'].append(petscconfvar+" requirement not met") 779 else: 780 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 781 continue 782 return testDict['SKIP'] == [] 783 784 def checkOutput(self,exfile,root,srcDict): 785 """ 786 Check and make sure the output files are in the output director 787 """ 788 debug=False 789 rpath=self.srcrelpath(root) 790 for test in srcDict: 791 if test in self.buildkeys: continue 792 if debug: print(rpath, exfile, test) 793 if 'output_file' in srcDict[test]: 794 output_file=srcDict[test]['output_file'] 795 else: 796 defroot = testparse.getDefaultOutputFileRoot(test) 797 if 'TODO' in srcDict[test]: continue 798 output_file="output/"+defroot+".out" 799 800 fullout=os.path.join(root,output_file) 801 if debug: print("---> ",fullout) 802 if not os.path.exists(fullout): 803 self.missing_files.append(fullout) 804 805 return 806 807 def genPetscTests_summarize(self,dataDict): 808 """ 809 Required method to state what happened 810 """ 811 if not self.summarize: return 812 indent=" " 813 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 814 fh=open(fhname,"w") 815 for root in dataDict: 816 relroot=self.srcrelpath(root) 817 pkg=relroot.split("/")[1] 818 fh.write(relroot+"\n") 819 allSrcs=[] 820 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 821 for exfile in dataDict[root]: 822 # Basic information 823 rfile=os.path.join(relroot,exfile) 824 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 825 fh.write(indent+exfile+indent*4+builtStatus+"\n") 826 827 for test in dataDict[root][exfile]: 828 if test in self.buildkeys: continue 829 line=indent*2+test 830 fh.write(line+"\n") 831 # Looks nice to have the keys in order 832 #for key in dataDict[root][exfile][test]: 833 for key in "isrun abstracted nsize args requires script".split(): 834 if key not in dataDict[root][exfile][test]: continue 835 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 836 fh.write(line+"\n") 837 fh.write("\n") 838 fh.write("\n") 839 fh.write("\n") 840 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 841 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 842 fh.close() 843 return 844 845 def genPetscTests(self,root,dirs,files,dataDict): 846 """ 847 Go through and parse the source files in the directory to generate 848 the examples based on the metadata contained in the source files 849 """ 850 debug=False 851 # Use examplesAnalyze to get what the makefles think are sources 852 #self.examplesAnalyze(root,dirs,files,anlzDict) 853 854 dataDict[root]={} 855 856 for exfile in files: 857 #TST: Until we replace files, still leaving the orginals as is 858 #if not exfile.startswith("new_"+"ex"): continue 859 #if not exfile.startswith("ex"): continue 860 861 # Ignore emacs and other temporary files 862 if exfile.startswith("."): continue 863 if exfile.startswith("#"): continue 864 if exfile.endswith("~"): continue 865 # Only parse source files 866 ext=os.path.splitext(exfile)[-1].lstrip('.') 867 if ext not in LANGS: continue 868 869 # Convenience 870 fullex=os.path.join(root,exfile) 871 if self.verbose: print(' --> '+fullex) 872 dataDict[root].update(testparse.parseTestFile(fullex,0)) 873 if exfile in dataDict[root]: 874 if not self.check_output: 875 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 876 else: 877 self.checkOutput(exfile,root,dataDict[root][exfile]) 878 879 return 880 881 def walktree(self,top): 882 """ 883 Walk a directory tree, starting from 'top' 884 """ 885 if self.check_output: 886 print("Checking for missing output files") 887 self.missing_files=[] 888 889 # Goal of action is to fill this dictionary 890 dataDict={} 891 for root, dirs, files in os.walk(top, topdown=True): 892 dirs.sort() 893 files.sort() 894 if not "examples" in root: continue 895 if "dSYM" in root: continue 896 if os.path.basename(root.rstrip("/")) == 'output': continue 897 if self.verbose: print(root) 898 self.genPetscTests(root,dirs,files,dataDict) 899 900 # If checking output, report results 901 if self.check_output: 902 if self.missing_files: 903 for file in set(self.missing_files): # set uniqifies 904 print(file) 905 sys.exit(1) 906 907 # Now summarize this dictionary 908 if self.verbose: self.genPetscTests_summarize(dataDict) 909 return dataDict 910 911 def gen_gnumake(self, fd): 912 """ 913 Overwrite of the method in the base PETSc class 914 """ 915 def write(stem, srcs): 916 for lang in LANGS: 917 if srcs[lang]['srcs']: 918 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 919 for pkg in self.pkg_pkgs: 920 srcs = self.gen_pkg(pkg) 921 write('testsrcs-' + pkg, srcs) 922 # Handle dependencies 923 for lang in LANGS: 924 for exfile in srcs[lang]['srcs']: 925 if exfile in srcs[lang]: 926 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 927 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 928 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 929 if deps: 930 # The executable literally depends on the object file because it is linked 931 fd.write(ex +": " + " ".join(deps) +'\n') 932 # The object file containing 'main' does not normally depend on other object 933 # files, but it does when it includes their modules. This dependency is 934 # overly blunt and could be reduced to only depend on object files for 935 # modules that are used, like "*f90aux.o". 936 fd.write(exfo +": " + " ".join(deps) +'\n') 937 938 return self.gendeps 939 940 def gen_pkg(self, pkg): 941 """ 942 Overwrite of the method in the base PETSc class 943 """ 944 return self.sources[pkg] 945 946 def write_gnumake(self, dataDict, output=None): 947 """ 948 Write out something similar to files from gmakegen.py 949 950 Test depends on script which also depends on source 951 file, but since I don't have a good way generating 952 acting on a single file (oops) just depend on 953 executable which in turn will depend on src file 954 """ 955 # Different options for how to set up the targets 956 compileExecsFirst=False 957 958 # Open file 959 fd = open(output, 'w') 960 961 # Write out the sources 962 gendeps = self.gen_gnumake(fd) 963 964 # Write out the tests and execname targets 965 fd.write("\n#Tests and executables\n") # Delimiter 966 967 for pkg in self.pkg_pkgs: 968 # These grab the ones that are built 969 for lang in LANGS: 970 testdeps=[] 971 for ftest in self.tests[pkg][lang]: 972 test=os.path.basename(ftest) 973 basedir=os.path.dirname(ftest) 974 testdeps.append(nameSpace(test,basedir)) 975 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 976 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 977 978 # test targets 979 for ftest in self.tests[pkg][lang]: 980 test=os.path.basename(ftest) 981 basedir=os.path.dirname(ftest) 982 testdir="${TESTDIR}/"+basedir+"/" 983 nmtest=nameSpace(test,basedir) 984 rundir=os.path.join(testdir,test) 985 script=test+".sh" 986 987 # Deps 988 exfile=self.tests[pkg][lang][ftest]['exfile'] 989 fullex=os.path.join(self.srcdir,exfile) 990 localexec=self.tests[pkg][lang][ftest]['exec'] 991 execname=os.path.join(testdir,localexec) 992 fullscript=os.path.join(testdir,script) 993 tmpfile=os.path.join(testdir,test,test+".tmp") 994 995 # *.counts depends on the script and either executable (will 996 # be run) or the example source file (SKIP or TODO) 997 fd.write('%s.counts : %s %s' 998 % (os.path.join('$(TESTDIR)/counts', nmtest), 999 fullscript, 1000 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1001 ) 1002 if exfile in self.sources[pkg][lang]: 1003 for dep in self.sources[pkg][lang][exfile]: 1004 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1005 fd.write('\n') 1006 1007 # Now write the args: 1008 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1009 1010 fd.close() 1011 return 1012 1013 def write_db(self, dataDict, testdir): 1014 """ 1015 Write out the dataDict into a pickle file 1016 """ 1017 fd = open(os.path.join(testdir,'datatest.pkl'), 'wb') 1018 pickle.dump(dataDict,fd) 1019 fd.close() 1020 return 1021 1022def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1023 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1024 srcdir=None, testdir=None, check=False): 1025 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1026 testdir=os.path.normpath(testdir) 1027 if petsc_arch: 1028 petsc_arch=petsc_arch.rstrip(os.path.sep) 1029 if len(petsc_arch.split(os.path.sep))>1: 1030 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1031 output = os.path.join(testdir, 'testfiles') 1032 1033 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1034 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1035 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1036 testdir=testdir,check=check) 1037 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1038 if not pEx.check_output: 1039 pEx.write_gnumake(dataDict, output) 1040 pEx.write_db(dataDict, testdir) 1041 1042if __name__ == '__main__': 1043 import optparse 1044 parser = optparse.OptionParser() 1045 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1046 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1047 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1048 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1049 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1050 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1051 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1052 help='Check whether output files are in output director') 1053 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1054 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1055 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1056 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1057 1058 opts, extra_args = parser.parse_args() 1059 if extra_args: 1060 import sys 1061 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1062 exit(1) 1063 if opts.testdir is None: 1064 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1065 1066 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1067 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1068 verbose=opts.verbose, 1069 single_ex=opts.single_executable, srcdir=opts.srcdir, 1070 testdir=opts.testdir, check=opts.check_output) 1071