1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 if self.inInstallDir: 103 # Case 2 discussed above 104 # set PETSC_ARCH to install directory to allow script to work in both 105 dirlist=thisscriptdir.split(os.path.sep) 106 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 107 self.arch_dir=installdir 108 if self.srcdir is None: 109 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 110 else: 111 if petsc_arch == '': 112 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 113 # Case 1 discussed above 114 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 115 if self.srcdir is None: 116 self.srcdir=os.path.join(self.petsc_dir,'src') 117 118 self.testroot_dir=os.path.abspath(testdir) 119 120 self.verbose=verbose 121 # Whether to write out a useful debugging 122 self.summarize=True if verbose else False 123 124 # For help in setting the requirements 125 self.precision_types="single double __float128 int32".split() 126 self.integer_types="int32 int64 long32 long64".split() 127 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 128 129 # Things that are not test 130 self.buildkeys=testparse.buildkeys 131 132 # Adding a dictionary for storing sources, objects, and tests 133 # to make building the dependency tree easier 134 self.sources={} 135 self.objects={} 136 self.tests={} 137 for pkg in self.pkg_pkgs: 138 self.sources[pkg]={} 139 self.objects[pkg]=[] 140 self.tests[pkg]={} 141 for lang in LANGS: 142 self.sources[pkg][lang]={} 143 self.sources[pkg][lang]['srcs']=[] 144 self.tests[pkg][lang]={} 145 146 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 147 148 self.indent=" " 149 if self.verbose: print('Finishing the constructor') 150 return 151 152 def srcrelpath(self,rdir): 153 """ 154 Get relative path to source directory 155 """ 156 return os.path.relpath(rdir,self.srcdir) 157 158 def getInInstallDir(self,thisscriptdir): 159 """ 160 When petsc is installed then this file in installed in: 161 <PREFIX>/share/petsc/examples/config/gmakegentest.py 162 otherwise the path is: 163 <PETSC_DIR>/config/gmakegentest.py 164 We use this difference to determine if we are in installdir 165 """ 166 dirlist=thisscriptdir.split(os.path.sep) 167 if len(dirlist)>4: 168 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 169 if lastfour==os.path.join('share','petsc','examples','config'): 170 return True 171 else: 172 return False 173 else: 174 return False 175 176 def getLanguage(self,srcfile): 177 """ 178 Based on the source, determine associated language as found in gmakegen.LANGS 179 Can we just return srcext[1:\] now? 180 """ 181 langReq=None 182 srcext=os.path.splitext(srcfile)[-1] 183 if srcext in ".F90".split(): langReq="F90" 184 if srcext in ".F".split(): langReq="F" 185 if srcext in ".cxx".split(): langReq="cxx" 186 if srcext in ".cpp".split(): langReq="cpp" 187 if srcext == ".cu": langReq="cu" 188 if srcext == ".c": langReq="c" 189 #if not langReq: print("ERROR: ", srcext, srcfile) 190 return langReq 191 192 def _getLoopVars(self,inDict,testname, isSubtest=False): 193 """ 194 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 195 Return: 196 inDict['args']: -ksp_monitor 197 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 198 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 199 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 200 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 201 subst should be passed in instead of inDict 202 """ 203 loopVars={}; newargs=[] 204 lsuffix='+' 205 argregex = re.compile(' (?=-[a-zA-Z])') 206 from testparse import parseLoopArgs 207 for key in inDict: 208 if key in ('SKIP', 'regexes'): 209 continue 210 akey=('subargs' if key=='args' else key) # what to assign 211 if akey not in inDict: inDict[akey]='' 212 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 213 # Always generate a loop over nsize, even if there is only one value 214 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 215 keystr = str(inDict[key]) 216 varlist = [] 217 for varset in argregex.split(keystr): 218 if not varset.strip(): continue 219 if '{{' in varset: 220 keyvar,lvars,ftype=parseLoopArgs(varset) 221 if akey not in loopVars: loopVars[akey]={} 222 varlist.append(keyvar) 223 loopVars[akey][keyvar]=[keyvar,lvars] 224 if akey=='nsize': 225 if len(lvars.split()) > 1: 226 lsuffix += akey +'-${' + keyvar + '}' 227 else: 228 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 229 lsuffix+=keyvar+'-${' + keyvar + '}_' 230 else: 231 if key=='args': 232 newargs.append(varset.strip()) 233 if varlist: 234 loopVars[akey]['varlist']=varlist 235 236 # For subtests, args are always substituted in (not top level) 237 if isSubtest: 238 inDict['subargs'] += " "+" ".join(newargs) 239 inDict['args']='' 240 if 'label_suffix' in inDict: 241 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 242 else: 243 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 244 else: 245 if loopVars: 246 inDict['args'] = ' '.join(newargs) 247 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 248 return loopVars 249 250 def getArgLabel(self,testDict): 251 """ 252 In all of the arguments in the test dictionary, create a simple 253 string for searching within the makefile system. For simplicity in 254 search, remove "-", for strings, etc. 255 Also, concatenate the arg commands 256 For now, ignore nsize -- seems hard to search for anyway 257 """ 258 # Collect all of the args associated with a test 259 argStr=("" if 'args' not in testDict else testDict['args']) 260 if 'subtests' in testDict: 261 for stest in testDict["subtests"]: 262 sd=testDict[stest] 263 argStr=argStr+("" if 'args' not in sd else sd['args']) 264 265 # Now go through and cleanup 266 argStr=re.sub('{{(.*?)}}',"",argStr) 267 argStr=re.sub('-'," ",argStr) 268 for digit in string.digits: argStr=re.sub(digit," ",argStr) 269 argStr=re.sub("\.","",argStr) 270 argStr=re.sub(",","",argStr) 271 argStr=re.sub('\+',' ',argStr) 272 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 273 return argStr.strip() 274 275 def addToSources(self,exfile,rpath,srcDict): 276 """ 277 Put into data structure that allows easy generation of makefile 278 """ 279 pkg=rpath.split(os.path.sep)[0] 280 relpfile=os.path.join(rpath,exfile) 281 lang=self.getLanguage(exfile) 282 if not lang: return 283 if pkg not in self.sources: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=rpath+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,rpath,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 pkg=rpath.split("/")[0] 304 nmtest=os.path.join(rpath,test) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 if pkg not in self.tests: return 308 self.tests[pkg][lang][nmtest]={} 309 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 310 self.tests[pkg][lang][nmtest]['exec']=execname 311 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 312 return 313 314 def getExecname(self,exfile,rpath): 315 """ 316 Generate bash script using template found next to this file. 317 This file is read in at constructor time to avoid file I/O 318 """ 319 if self.single_ex: 320 execname=rpath.split("/")[1]+"-ex" 321 else: 322 execname=os.path.splitext(exfile)[0] 323 return execname 324 325 def getSubstVars(self,testDict,rpath,testname): 326 """ 327 Create a dictionary with all of the variables that get substituted 328 into the template commands found in example_template.py 329 """ 330 subst={} 331 332 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 333 if 'nsize' not in testDict: testDict['nsize'] = '1' 334 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 335 for ak in testparse.acceptedkeys: 336 if ak=='test': continue 337 subst[ak]=(testDict[ak] if ak in testDict else '') 338 339 # Now do other variables 340 subst['execname']=testDict['execname'] 341 subst['error']='' 342 if 'filter' in testDict: 343 if testDict['filter'].startswith("Error:"): 344 subst['error']="Error" 345 subst['filter']=testDict['filter'].lstrip("Error:") 346 else: 347 subst['filter']=testDict['filter'] 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(self.srcdir, rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['pkg_name']=self.pkg_name 371 subst['pkg_dir']=self.pkg_dir 372 subst['pkg_arch']=self.petsc_arch 373 subst['CONFIG_DIR']=thisscriptdir 374 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 375 subst['diff']=self.conf['DIFF'] 376 subst['rm']=self.conf['RM'] 377 subst['grep']=self.conf['GREP'] 378 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 379 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 380 381 # Output file is special because of subtests override 382 defroot = testparse.getDefaultOutputFileRoot(testname) 383 if 'output_file' not in testDict: 384 subst['output_file']="output/"+defroot+".out" 385 subst['redirect_file']=defroot+".tmp" 386 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 387 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if 'altfiles' not in subst: 447 cmd=diffindnt+self._substVars(subst,example_template.difftest) 448 else: 449 # Have to do it by hand a bit because of variable number of alt files 450 rf=subst['redirect_file'] 451 cmd=diffindnt+example_template.difftest.split('@')[0] 452 for i in range(len(subst['altfiles'])): 453 af=subst['altfiles'][i] 454 cmd+=af+' '+rf 455 if i!=len(subst['altfiles'])-1: 456 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 457 cmd+=' || ${diff_exe} ' 458 else: 459 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 460 cmd+=subst['label_suffix']+' ""' # Quotes are painful 461 cmdLines+=cmd+"\n" 462 cmdLines+=cmdindnt+'else\n' 463 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 464 cmdLines+=cmdindnt+'fi\n' 465 return cmdLines 466 467 def _writeTodoSkip(self,fh,tors,reasons,footer): 468 """ 469 Write out the TODO and SKIP lines in the file 470 The TODO or SKIP variable, tors, should be lower case 471 """ 472 TORS=tors.upper() 473 template=eval("example_template."+tors+"line") 474 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 475 tab = '' 476 if reasons: 477 fh.write('if ! $force; then\n') 478 tab = tab + ' ' 479 if reasons == ["Requires DATAFILESPATH"]: 480 # The only reason not to run is DATAFILESPATH, which we check at run-time 481 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 482 tab = tab + ' ' 483 if reasons: 484 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 485 fh.write(tab+footer+"\n") 486 fh.write(tab+"exit\n") 487 if reasons == ["Requires DATAFILESPATH"]: 488 fh.write(' fi\n') 489 if reasons: 490 fh.write('fi\n') 491 fh.write('\n\n') 492 return 493 494 def getLoopVarsHead(self,loopVars,i,usedVars={}): 495 """ 496 Generate a nicely indented string with the format loops 497 Here is what the data structure looks like 498 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 499 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 500 loopVars['subargs']['pc_type']=["j","cholesky sor"] 501 """ 502 outstr=''; indnt=self.indent 503 504 for key in loopVars: 505 if key in usedVars: continue # Do not duplicate setting vars 506 for var in loopVars[key]['varlist']: 507 varval=loopVars[key][var] 508 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 509 outstr += "\n\n" 510 511 for key in loopVars: 512 for var in loopVars[key]['varlist']: 513 varval=loopVars[key][var] 514 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 515 i = i + 1 516 return (outstr,i) 517 518 def getLoopVarsFoot(self,loopVars,i): 519 outstr=''; indnt=self.indent 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 i = i - 1 523 outstr += indnt * i + "done\n" 524 return (outstr,i) 525 526 def genRunScript(self,testname,root,isRun,srcDict): 527 """ 528 Generate bash script using template found next to this file. 529 This file is read in at constructor time to avoid file I/O 530 """ 531 # runscript_dir directory has to be consistent with gmakefile 532 testDict=srcDict[testname] 533 rpath=self.srcrelpath(root) 534 runscript_dir=os.path.join(self.testroot_dir,rpath) 535 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 536 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 537 538 # Get variables to go into shell scripts. last time testDict used 539 subst=self.getSubstVars(testDict,rpath,testname) 540 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 541 if 'subtests' in testDict: 542 # The subtests inherit inDict, so we don't need top-level loops. 543 loopVars = {} 544 545 #Handle runfiles 546 for lfile in subst.get('localrunfiles','').split(): 547 install_files(os.path.join(root, lfile), 548 os.path.join(runscript_dir, os.path.dirname(lfile))) 549 # Check subtests for local runfiles 550 for stest in subst.get("subtests",[]): 551 for lfile in testDict[stest].get('localrunfiles','').split(): 552 install_files(os.path.join(root, lfile), 553 os.path.join(runscript_dir, os.path.dirname(lfile))) 554 555 # Now substitute the key variables into the header and footer 556 header=self._substVars(subst,example_template.header) 557 # The header is done twice to enable @...@ in header 558 header=self._substVars(subst,header) 559 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 560 561 # Start writing the file 562 fh.write(header+"\n") 563 564 # If there is a TODO or a SKIP then we do it before writing out the 565 # rest of the command (which is useful for working on the test) 566 # SKIP and TODO can be for the source file or for the runs 567 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 568 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 569 570 j=0 # for indentation 571 572 if loopVars: 573 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 574 if (loopHead): fh.write(loopHead+"\n") 575 576 # Subtests are special 577 allLoopVars=list(loopVars.keys()) 578 if 'subtests' in testDict: 579 substP=subst # Subtests can inherit args but be careful 580 k=0 # for label suffixes 581 for stest in testDict["subtests"]: 582 subst=substP.copy() 583 subst.update(testDict[stest]) 584 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 585 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 586 if sLoopVars: 587 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 588 allLoopVars+=list(sLoopVars.keys()) 589 fh.write(sLoopHead+"\n") 590 fh.write(self.getCmds(subst,j)+"\n") 591 if sLoopVars: 592 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 593 fh.write(sLoopFoot+"\n") 594 else: 595 fh.write(self.getCmds(subst,j)+"\n") 596 597 if loopVars: 598 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 599 fh.write(loopFoot+"\n") 600 601 fh.write(footer+"\n") 602 603 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 604 #if '10_9' in testname: sys.exit() 605 return 606 607 def genScriptsAndInfo(self,exfile,root,srcDict): 608 """ 609 Generate scripts from the source file, determine if built, etc. 610 For every test in the exfile with info in the srcDict: 611 1. Determine if it needs to be run for this arch 612 2. Generate the script 613 3. Generate the data needed to write out the makefile in a 614 convenient way 615 All tests are *always* run, but some may be SKIP'd per the TAP standard 616 """ 617 debug=False 618 rpath=self.srcrelpath(root) 619 execname=self.getExecname(exfile,rpath) 620 isBuilt=self._isBuilt(exfile,srcDict) 621 for test in srcDict: 622 if test in self.buildkeys: continue 623 if debug: print(nameSpace(exfile,root), test) 624 srcDict[test]['execname']=execname # Convenience in generating scripts 625 isRun=self._isRun(srcDict[test]) 626 self.genRunScript(test,root,isRun,srcDict) 627 srcDict[test]['isrun']=isRun 628 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 629 630 # This adds to datastructure for building deps 631 if isBuilt: self.addToSources(exfile,rpath,srcDict) 632 return 633 634 def _isBuilt(self,exfile,srcDict): 635 """ 636 Determine if this file should be built. 637 """ 638 # Get the language based on file extension 639 srcDict['SKIP'] = [] 640 lang=self.getLanguage(exfile) 641 if (lang=="F" or lang=="F90"): 642 if not self.have_fortran: 643 srcDict["SKIP"].append("Fortran required for this test") 644 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 645 srcDict["SKIP"].append("Fortran f90freeform required for this test") 646 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 647 srcDict["SKIP"].append("CUDA required for this test") 648 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 649 srcDict["SKIP"].append("C++ required for this test") 650 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 651 srcDict["SKIP"].append("C++ required for this test") 652 653 # Deprecated source files 654 if srcDict.get("TODO"): 655 return False 656 657 # isRun can work with srcDict to handle the requires 658 if "requires" in srcDict: 659 if srcDict["requires"]: 660 return self._isRun(srcDict) 661 662 return srcDict['SKIP'] == [] 663 664 665 def _isRun(self,testDict, debug=False): 666 """ 667 Based on the requirements listed in the src file and the petscconf.h 668 info, determine whether this test should be run or not. 669 """ 670 indent=" " 671 672 if 'SKIP' not in testDict: 673 testDict['SKIP'] = [] 674 # MPI requirements 675 if 'MPI_IS_MPIUNI' in self.conf: 676 if testDict.get('nsize', '1') != '1': 677 testDict['SKIP'].append("Parallel test with serial build") 678 679 # The requirements for the test are the sum of all the run subtests 680 if 'subtests' in testDict: 681 if 'requires' not in testDict: testDict['requires']="" 682 for stest in testDict['subtests']: 683 if 'requires' in testDict[stest]: 684 testDict['requires']+=" "+testDict[stest]['requires'] 685 if testDict[stest].get('nsize', '1') != '1': 686 testDict['SKIP'].append("Parallel test with serial build") 687 break 688 689 # Now go through all requirements 690 if 'requires' in testDict: 691 for requirement in testDict['requires'].split(): 692 requirement=requirement.strip() 693 if not requirement: continue 694 if debug: print(indent+"Requirement: ", requirement) 695 isNull=False 696 if requirement.startswith("!"): 697 requirement=requirement[1:]; isNull=True 698 # Precision requirement for reals 699 if requirement in self.precision_types: 700 if self.conf['PETSC_PRECISION']==requirement: 701 if isNull: 702 testDict['SKIP'].append("not "+requirement+" required") 703 continue 704 continue # Success 705 elif not isNull: 706 testDict['SKIP'].append(requirement+" required") 707 continue 708 # Precision requirement for ints 709 if requirement in self.integer_types: 710 if requirement=="int32": 711 if self.conf['PETSC_SIZEOF_INT']==4: 712 if isNull: 713 testDict['SKIP'].append("not int32 required") 714 continue 715 continue # Success 716 elif not isNull: 717 testDict['SKIP'].append("int32 required") 718 continue 719 if requirement=="int64": 720 if self.conf['PETSC_SIZEOF_INT']==8: 721 if isNull: 722 testDict['SKIP'].append("NOT int64 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int64 required") 727 continue 728 if requirement.startswith("long"): 729 reqsize = int(requirement[4:])//8 730 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 731 if longsize==reqsize: 732 if isNull: 733 testDict['SKIP'].append("not %s required" % requirement) 734 continue 735 continue # Success 736 elif not isNull: 737 testDict['SKIP'].append("%s required" % requirement) 738 continue 739 # Datafilespath 740 if requirement=="datafilespath" and not isNull: 741 testDict['SKIP'].append("Requires DATAFILESPATH") 742 continue 743 # Defines -- not sure I have comments matching 744 if "define(" in requirement.lower(): 745 reqdef=requirement.split("(")[1].split(")")[0] 746 if reqdef in self.conf: 747 if isNull: 748 testDict['SKIP'].append("Null requirement not met: "+requirement) 749 continue 750 continue # Success 751 elif not isNull: 752 testDict['SKIP'].append("Required: "+requirement) 753 continue 754 755 # Rest should be packages that we can just get from conf 756 if requirement in ["complex","debug"]: 757 petscconfvar="PETSC_USE_"+requirement.upper() 758 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 759 else: 760 petscconfvar="PETSC_HAVE_"+requirement.upper() 761 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 762 petsccv = self.conf.get(petscconfvar) 763 pkgcv = self.conf.get(pkgconfvar) 764 765 if petsccv or pkgcv: 766 if isNull: 767 if petsccv: 768 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 769 continue 770 else: 771 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 772 continue 773 continue # Success 774 elif not isNull: 775 if not petsccv and not pkgcv: 776 if debug: print("requirement not found: ", requirement) 777 if self.pkg_name == 'petsc': 778 testDict['SKIP'].append(petscconfvar+" requirement not met") 779 else: 780 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 781 continue 782 return testDict['SKIP'] == [] 783 784 def checkOutput(self,exfile,root,srcDict): 785 """ 786 Check and make sure the output files are in the output director 787 """ 788 debug=False 789 rpath=self.srcrelpath(root) 790 for test in srcDict: 791 if test in self.buildkeys: continue 792 if debug: print(rpath, exfile, test) 793 if 'output_file' in srcDict[test]: 794 output_file=srcDict[test]['output_file'] 795 else: 796 defroot = testparse.getDefaultOutputFileRoot(test) 797 if 'TODO' in srcDict[test]: continue 798 output_file="output/"+defroot+".out" 799 800 fullout=os.path.join(root,output_file) 801 if debug: print("---> ",fullout) 802 if not os.path.exists(fullout): 803 self.missing_files.append(fullout) 804 805 return 806 807 def genPetscTests_summarize(self,dataDict): 808 """ 809 Required method to state what happened 810 """ 811 if not self.summarize: return 812 indent=" " 813 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 814 with open(fhname, "w") as fh: 815 for root in dataDict: 816 relroot=self.srcrelpath(root) 817 pkg=relroot.split("/")[1] 818 fh.write(relroot+"\n") 819 allSrcs=[] 820 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 821 for exfile in dataDict[root]: 822 # Basic information 823 rfile=os.path.join(relroot,exfile) 824 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 825 fh.write(indent+exfile+indent*4+builtStatus+"\n") 826 for test in dataDict[root][exfile]: 827 if test in self.buildkeys: continue 828 line=indent*2+test 829 fh.write(line+"\n") 830 # Looks nice to have the keys in order 831 #for key in dataDict[root][exfile][test]: 832 for key in "isrun abstracted nsize args requires script".split(): 833 if key not in dataDict[root][exfile][test]: continue 834 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 835 fh.write(line+"\n") 836 fh.write("\n") 837 fh.write("\n") 838 fh.write("\n") 839 return 840 841 def genPetscTests(self,root,dirs,files,dataDict): 842 """ 843 Go through and parse the source files in the directory to generate 844 the examples based on the metadata contained in the source files 845 """ 846 debug=False 847 # Use examplesAnalyze to get what the makefles think are sources 848 #self.examplesAnalyze(root,dirs,files,anlzDict) 849 850 dataDict[root]={} 851 852 for exfile in files: 853 #TST: Until we replace files, still leaving the orginals as is 854 #if not exfile.startswith("new_"+"ex"): continue 855 #if not exfile.startswith("ex"): continue 856 857 # Ignore emacs and other temporary files 858 if exfile.startswith("."): continue 859 if exfile.startswith("#"): continue 860 if exfile.endswith("~"): continue 861 # Only parse source files 862 ext=os.path.splitext(exfile)[-1].lstrip('.') 863 if ext not in LANGS: continue 864 865 # Convenience 866 fullex=os.path.join(root,exfile) 867 if self.verbose: print(' --> '+fullex) 868 dataDict[root].update(testparse.parseTestFile(fullex,0)) 869 if exfile in dataDict[root]: 870 if not self.check_output: 871 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 872 else: 873 self.checkOutput(exfile,root,dataDict[root][exfile]) 874 875 return 876 877 def walktree(self,top): 878 """ 879 Walk a directory tree, starting from 'top' 880 """ 881 if self.check_output: 882 print("Checking for missing output files") 883 self.missing_files=[] 884 885 # Goal of action is to fill this dictionary 886 dataDict={} 887 for root, dirs, files in os.walk(top, topdown=True): 888 dirs.sort() 889 files.sort() 890 if "/tests" not in root and "/tutorials" not in root: continue 891 if "dSYM" in root: continue 892 if os.path.basename(root.rstrip("/")) == 'output': continue 893 if self.verbose: print(root) 894 self.genPetscTests(root,dirs,files,dataDict) 895 896 # If checking output, report results 897 if self.check_output: 898 if self.missing_files: 899 for file in set(self.missing_files): # set uniqifies 900 print(file) 901 sys.exit(1) 902 903 # Now summarize this dictionary 904 if self.verbose: self.genPetscTests_summarize(dataDict) 905 return dataDict 906 907 def gen_gnumake(self, fd): 908 """ 909 Overwrite of the method in the base PETSc class 910 """ 911 def write(stem, srcs): 912 for lang in LANGS: 913 if srcs[lang]['srcs']: 914 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 915 for pkg in self.pkg_pkgs: 916 srcs = self.gen_pkg(pkg) 917 write('testsrcs-' + pkg, srcs) 918 # Handle dependencies 919 for lang in LANGS: 920 for exfile in srcs[lang]['srcs']: 921 if exfile in srcs[lang]: 922 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 923 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 924 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 925 if deps: 926 # The executable literally depends on the object file because it is linked 927 fd.write(ex +": " + " ".join(deps) +'\n') 928 # The object file containing 'main' does not normally depend on other object 929 # files, but it does when it includes their modules. This dependency is 930 # overly blunt and could be reduced to only depend on object files for 931 # modules that are used, like "*f90aux.o". 932 fd.write(exfo +": " + " ".join(deps) +'\n') 933 934 return self.gendeps 935 936 def gen_pkg(self, pkg): 937 """ 938 Overwrite of the method in the base PETSc class 939 """ 940 return self.sources[pkg] 941 942 def write_gnumake(self, dataDict, output=None): 943 """ 944 Write out something similar to files from gmakegen.py 945 946 Test depends on script which also depends on source 947 file, but since I don't have a good way generating 948 acting on a single file (oops) just depend on 949 executable which in turn will depend on src file 950 """ 951 # Different options for how to set up the targets 952 compileExecsFirst=False 953 954 # Open file 955 with open(output, 'w') as fd: 956 # Write out the sources 957 gendeps = self.gen_gnumake(fd) 958 959 # Write out the tests and execname targets 960 fd.write("\n#Tests and executables\n") # Delimiter 961 962 for pkg in self.pkg_pkgs: 963 # These grab the ones that are built 964 for lang in LANGS: 965 testdeps=[] 966 for ftest in self.tests[pkg][lang]: 967 test=os.path.basename(ftest) 968 basedir=os.path.dirname(ftest) 969 testdeps.append(nameSpace(test,basedir)) 970 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 971 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 972 973 # test targets 974 for ftest in self.tests[pkg][lang]: 975 test=os.path.basename(ftest) 976 basedir=os.path.dirname(ftest) 977 testdir="${TESTDIR}/"+basedir+"/" 978 nmtest=nameSpace(test,basedir) 979 rundir=os.path.join(testdir,test) 980 script=test+".sh" 981 982 # Deps 983 exfile=self.tests[pkg][lang][ftest]['exfile'] 984 fullex=os.path.join(self.srcdir,exfile) 985 localexec=self.tests[pkg][lang][ftest]['exec'] 986 execname=os.path.join(testdir,localexec) 987 fullscript=os.path.join(testdir,script) 988 tmpfile=os.path.join(testdir,test,test+".tmp") 989 990 # *.counts depends on the script and either executable (will 991 # be run) or the example source file (SKIP or TODO) 992 fd.write('%s.counts : %s %s' 993 % (os.path.join('$(TESTDIR)/counts', nmtest), 994 fullscript, 995 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 996 ) 997 if exfile in self.sources[pkg][lang]: 998 for dep in self.sources[pkg][lang][exfile]: 999 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1000 fd.write('\n') 1001 1002 # Now write the args: 1003 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1004 1005 return 1006 1007 def write_db(self, dataDict, testdir): 1008 """ 1009 Write out the dataDict into a pickle file 1010 """ 1011 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1012 pickle.dump(dataDict,fd) 1013 return 1014 1015def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1016 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1017 srcdir=None, testdir=None, check=False): 1018 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1019 testdir=os.path.normpath(testdir) 1020 if petsc_arch: 1021 petsc_arch=petsc_arch.rstrip(os.path.sep) 1022 if len(petsc_arch.split(os.path.sep))>1: 1023 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1024 output = os.path.join(testdir, 'testfiles') 1025 1026 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1027 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1028 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1029 testdir=testdir,check=check) 1030 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1031 if not pEx.check_output: 1032 pEx.write_gnumake(dataDict, output) 1033 pEx.write_db(dataDict, testdir) 1034 1035if __name__ == '__main__': 1036 import optparse 1037 parser = optparse.OptionParser() 1038 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1039 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1040 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1041 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1042 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1043 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1044 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1045 help='Check whether output files are in output director') 1046 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1047 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1048 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1049 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1050 1051 opts, extra_args = parser.parse_args() 1052 if extra_args: 1053 import sys 1054 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1055 exit(1) 1056 if opts.testdir is None: 1057 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1058 1059 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1060 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1061 verbose=opts.verbose, 1062 single_ex=opts.single_executable, srcdir=opts.srcdir, 1063 testdir=opts.testdir, check=opts.check_output) 1064