1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from collections import defaultdict 10from gmakegen import * 11 12import inspect 13thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 14sys.path.insert(0,thisscriptdir) 15import testparse 16import example_template 17 18 19""" 20 21There are 2 modes of running tests: Normal builds and run from prefix of 22install. They affect where to find things: 23 24 25Case 1. Normal builds: 26 27 +---------------------+----------------------------------+ 28 | PETSC_DIR | <git dir> | 29 +---------------------+----------------------------------+ 30 | PETSC_ARCH | arch-foo | 31 +---------------------+----------------------------------+ 32 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 33 +---------------------+----------------------------------+ 34 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 35 +---------------------+----------------------------------+ 36 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 37 +---------------------+----------------------------------+ 38 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 41 +---------------------+----------------------------------+ 42 43 44Case 2. From install dir: 45 46 +---------------------+-------------------------------------------------------+ 47 | PETSC_DIR | <prefix dir> | 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_ARCH | '' | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 60 +---------------------+-------------------------------------------------------+ 61 62""" 63 64def install_files(source, destdir): 65 """Install file or directory 'source' to 'destdir'. Does not preserve 66 mode (permissions). 67 """ 68 if not os.path.isdir(destdir): 69 os.makedirs(destdir) 70 if os.path.isdir(source): 71 for name in os.listdir(source): 72 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 73 else: 74 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 75 76class generateExamples(Petsc): 77 """ 78 gmakegen.py has basic structure for finding the files, writing out 79 the dependencies, etc. 80 """ 81 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 82 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 83 84 self.single_ex=single_ex 85 self.srcdir=srcdir 86 87 # Set locations to handle movement 88 self.inInstallDir=self.getInInstallDir(thisscriptdir) 89 90 if self.inInstallDir: 91 # Case 2 discussed above 92 # set PETSC_ARCH to install directory to allow script to work in both 93 dirlist=thisscriptdir.split(os.path.sep) 94 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 95 self.arch_dir=installdir 96 if self.srcdir is None: 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 if self.srcdir is None: 104 self.srcdir=os.path.join(self.petsc_dir,'src') 105 106 self.testroot_dir=os.path.abspath(testdir) 107 108 self.ptNaming=True 109 self.verbose=verbose 110 # Whether to write out a useful debugging 111 self.summarize=True if verbose else False 112 113 # For help in setting the requirements 114 self.precision_types="single double __float128 int32".split() 115 self.integer_types="int32 int64 long32 long64".split() 116 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 117 118 # Things that are not test 119 self.buildkeys=testparse.buildkeys 120 121 # Adding a dictionary for storing sources, objects, and tests 122 # to make building the dependency tree easier 123 self.sources={} 124 self.objects={} 125 self.tests={} 126 for pkg in self.pkg_pkgs: 127 self.sources[pkg]={} 128 self.objects[pkg]=[] 129 self.tests[pkg]={} 130 for lang in LANGS: 131 self.sources[pkg][lang]={} 132 self.sources[pkg][lang]['srcs']=[] 133 self.tests[pkg][lang]={} 134 135 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 136 137 self.indent=" " 138 if self.verbose: print('Finishing the constructor') 139 return 140 141 def srcrelpath(self,rdir): 142 """ 143 Get relative path to source directory 144 """ 145 return os.path.relpath(rdir,self.srcdir) 146 147 def getInInstallDir(self,thisscriptdir): 148 """ 149 When petsc is installed then this file in installed in: 150 <PREFIX>/share/petsc/examples/config/gmakegentest.py 151 otherwise the path is: 152 <PETSC_DIR>/config/gmakegentest.py 153 We use this difference to determine if we are in installdir 154 """ 155 dirlist=thisscriptdir.split(os.path.sep) 156 if len(dirlist)>4: 157 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 158 if lastfour==os.path.join('share','petsc','examples','config'): 159 return True 160 else: 161 return False 162 else: 163 return False 164 165 def nameSpace(self,srcfile,srcdir): 166 """ 167 Because the scripts have a non-unique naming, the pretty-printing 168 needs to convey the srcdir and srcfile. There are two ways of doing this. 169 """ 170 if self.ptNaming: 171 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 172 cdir=srcdir 173 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 174 nameString=prefix+srcfile 175 else: 176 #nameString=srcdir+": "+srcfile 177 nameString=srcfile 178 return nameString 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext=os.path.splitext(srcfile)[-1] 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".cpp".split(): langReq="cpp" 191 if srcext == ".cu": langReq="cu" 192 if srcext == ".c": langReq="c" 193 #if not langReq: print("ERROR: ", srcext, srcfile) 194 return langReq 195 196 def _getLoopVars(self,inDict,testname, isSubtest=False): 197 """ 198 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 199 Return: 200 inDict['args']: -ksp_monitor 201 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 202 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 203 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 204 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 205 subst should be passed in instead of inDict 206 """ 207 loopVars={}; newargs=[] 208 lsuffix='_' 209 argregex = re.compile(' (?=-[a-zA-Z])') 210 from testparse import parseLoopArgs 211 for key in inDict: 212 if key in ('SKIP', 'regexes'): 213 continue 214 akey=('subargs' if key=='args' else key) # what to assign 215 if akey not in inDict: inDict[akey]='' 216 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 217 # Always generate a loop over nsize, even if there is only one value 218 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 219 keystr = str(inDict[key]) 220 varlist = [] 221 for varset in argregex.split(keystr): 222 if not varset.strip(): continue 223 if '{{' in varset: 224 keyvar,lvars,ftype=parseLoopArgs(varset) 225 if akey not in loopVars: loopVars[akey]={} 226 varlist.append(keyvar) 227 loopVars[akey][keyvar]=[keyvar,lvars] 228 if akey=='nsize': 229 if len(lvars.split()) > 1: 230 lsuffix += akey +'-${' + keyvar + '}' 231 else: 232 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 233 lsuffix+=keyvar+'-${' + keyvar + '}_' 234 else: 235 if key=='args': 236 newargs.append(varset.strip()) 237 if varlist: 238 loopVars[akey]['varlist']=varlist 239 240 # For subtests, args are always substituted in (not top level) 241 if isSubtest: 242 inDict['subargs'] += " ".join(newargs) 243 inDict['args']='' 244 if 'label_suffix' in inDict: 245 inDict['label_suffix']+=lsuffix.rstrip('_') 246 else: 247 inDict['label_suffix']=lsuffix.rstrip('_') 248 else: 249 if loopVars: 250 inDict['args'] = ' '.join(newargs) 251 inDict['label_suffix']=lsuffix.rstrip('_') 252 return loopVars 253 254 def getArgLabel(self,testDict): 255 """ 256 In all of the arguments in the test dictionary, create a simple 257 string for searching within the makefile system. For simplicity in 258 search, remove "-", for strings, etc. 259 Also, concatenate the arg commands 260 For now, ignore nsize -- seems hard to search for anyway 261 """ 262 # Collect all of the args associated with a test 263 argStr=("" if 'args' not in testDict else testDict['args']) 264 if 'subtests' in testDict: 265 for stest in testDict["subtests"]: 266 sd=testDict[stest] 267 argStr=argStr+("" if 'args' not in sd else sd['args']) 268 269 # Now go through and cleanup 270 argStr=re.sub('{{(.*?)}}',"",argStr) 271 argStr=re.sub('-'," ",argStr) 272 for digit in string.digits: argStr=re.sub(digit," ",argStr) 273 argStr=re.sub("\.","",argStr) 274 argStr=re.sub(",","",argStr) 275 argStr=re.sub('\+',' ',argStr) 276 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 277 return argStr.strip() 278 279 def addToSources(self,exfile,rpath,srcDict): 280 """ 281 Put into data structure that allows easy generation of makefile 282 """ 283 pkg=rpath.split(os.path.sep)[0] 284 relpfile=os.path.join(rpath,exfile) 285 lang=self.getLanguage(exfile) 286 if not lang: return 287 if pkg not in self.sources: return 288 self.sources[pkg][lang]['srcs'].append(relpfile) 289 self.sources[pkg][lang][relpfile] = [] 290 if 'depends' in srcDict: 291 depSrcList=srcDict['depends'].split() 292 for depSrc in depSrcList: 293 depObj=os.path.splitext(depSrc)[0]+".o" 294 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 295 296 # In gmakefile, ${TESTDIR} var specifies the object compilation 297 testsdir=rpath+"/" 298 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 299 self.objects[pkg].append(objfile) 300 return 301 302 def addToTests(self,test,rpath,exfile,execname,testDict): 303 """ 304 Put into data structure that allows easy generation of makefile 305 Organized by languages to allow testing of languages 306 """ 307 pkg=rpath.split("/")[0] 308 nmtest=os.path.join(rpath,test) 309 lang=self.getLanguage(exfile) 310 if not lang: return 311 if pkg not in self.tests: return 312 self.tests[pkg][lang][nmtest]={} 313 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 314 self.tests[pkg][lang][nmtest]['exec']=execname 315 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 316 return 317 318 def getExecname(self,exfile,rpath): 319 """ 320 Generate bash script using template found next to this file. 321 This file is read in at constructor time to avoid file I/O 322 """ 323 if self.single_ex: 324 execname=rpath.split("/")[1]+"-ex" 325 else: 326 execname=os.path.splitext(exfile)[0] 327 return execname 328 329 def getSubstVars(self,testDict,rpath,testname): 330 """ 331 Create a dictionary with all of the variables that get substituted 332 into the template commands found in example_template.py 333 """ 334 subst={} 335 336 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 337 if 'nsize' not in testDict: testDict['nsize'] = '1' 338 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 339 for ak in testparse.acceptedkeys: 340 if ak=='test': continue 341 subst[ak]=(testDict[ak] if ak in testDict else '') 342 343 # Now do other variables 344 subst['execname']=testDict['execname'] 345 if 'filter' in testDict: 346 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 347 348 # Others 349 subst['subargs']='' # Default. For variables override 350 subst['srcdir']=os.path.join(self.srcdir, rpath) 351 subst['label_suffix']='' 352 subst['comments']="\n#".join(subst['comments'].split("\n")) 353 if subst['comments']: subst['comments']="#"+subst['comments'] 354 subst['exec']="../"+subst['execname'] 355 subst['testroot']=self.testroot_dir 356 subst['testname']=testname 357 dp = self.conf.get('DATAFILESPATH','') 358 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 359 360 # This is used to label some matrices 361 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 362 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 363 364 #Conf vars 365 if self.petsc_arch.find('valgrind')>=0: 366 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 367 else: 368 subst['mpiexec']=self.conf['MPIEXEC'] 369 subst['pkg_name']=self.pkg_name 370 subst['pkg_dir']=self.pkg_dir 371 subst['pkg_arch']=self.petsc_arch 372 subst['CONFIG_DIR']=thisscriptdir 373 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 374 subst['diff']=self.conf['DIFF'] 375 subst['rm']=self.conf['RM'] 376 subst['grep']=self.conf['GREP'] 377 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 378 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 379 380 # Output file is special because of subtests override 381 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 382 if not "_" in defroot: defroot=defroot+"_1" 383 subst['defroot']=defroot 384 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 385 subst['redirect_file']=defroot+".tmp" 386 if 'output_file' not in testDict: 387 subst['output_file']="output/"+defroot+".out" 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/examples/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if not subst['filter_output']: 447 if 'altfiles' not in subst: 448 cmd=diffindnt+self._substVars(subst,example_template.difftest) 449 else: 450 # Have to do it by hand a bit because of variable number of alt files 451 rf=subst['redirect_file'] 452 cmd=diffindnt+example_template.difftest.split('@')[0] 453 for i in range(len(subst['altfiles'])): 454 af=subst['altfiles'][i] 455 cmd+=af+' '+rf 456 if i!=len(subst['altfiles'])-1: 457 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 458 cmd+=' || ${diff_exe} ' 459 else: 460 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 461 cmd+=subst['label_suffix']+' ""' # Quotes are painful 462 else: 463 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 464 cmdLines+=cmd+"\n" 465 cmdLines+=cmdindnt+'else\n' 466 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 467 cmdLines+=cmdindnt+'fi\n' 468 return cmdLines 469 470 def _writeTodoSkip(self,fh,tors,reasons,footer): 471 """ 472 Write out the TODO and SKIP lines in the file 473 The TODO or SKIP variable, tors, should be lower case 474 """ 475 TORS=tors.upper() 476 template=eval("example_template."+tors+"line") 477 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 478 tab = '' 479 if reasons: 480 fh.write('if ! $force; then\n') 481 tab = tab + ' ' 482 if reasons == ["Requires DATAFILESPATH"]: 483 # The only reason not to run is DATAFILESPATH, which we check at run-time 484 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 485 tab = tab + ' ' 486 if reasons: 487 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 488 fh.write(tab+footer+"\n") 489 fh.write(tab+"exit\n") 490 if reasons == ["Requires DATAFILESPATH"]: 491 fh.write(' fi\n') 492 if reasons: 493 fh.write('fi\n') 494 fh.write('\n\n') 495 return 496 497 def getLoopVarsHead(self,loopVars,i,usedVars={}): 498 """ 499 Generate a nicely indented string with the format loops 500 Here is what the data structure looks like 501 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 502 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 503 loopVars['subargs']['pc_type']=["j","cholesky sor"] 504 """ 505 outstr=''; indnt=self.indent 506 507 for key in loopVars: 508 if key in usedVars: continue # Do not duplicate setting vars 509 for var in loopVars[key]['varlist']: 510 varval=loopVars[key][var] 511 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 512 outstr += "\n\n" 513 514 for key in loopVars: 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 518 i = i + 1 519 return (outstr,i) 520 521 def getLoopVarsFoot(self,loopVars,i): 522 outstr=''; indnt=self.indent 523 for key in loopVars: 524 for var in loopVars[key]['varlist']: 525 i = i - 1 526 outstr += indnt * i + "done\n" 527 return (outstr,i) 528 529 def genRunScript(self,testname,root,isRun,srcDict): 530 """ 531 Generate bash script using template found next to this file. 532 This file is read in at constructor time to avoid file I/O 533 """ 534 # runscript_dir directory has to be consistent with gmakefile 535 testDict=srcDict[testname] 536 rpath=self.srcrelpath(root) 537 runscript_dir=os.path.join(self.testroot_dir,rpath) 538 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 539 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 540 541 # Get variables to go into shell scripts. last time testDict used 542 subst=self.getSubstVars(testDict,rpath,testname) 543 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 544 if 'subtests' in testDict: 545 # The subtests inherit inDict, so we don't need top-level loops. 546 loopVars = {} 547 548 #Handle runfiles 549 for lfile in subst.get('localrunfiles','').split(): 550 install_files(os.path.join(root, lfile), 551 os.path.join(runscript_dir, os.path.dirname(lfile))) 552 # Check subtests for local runfiles 553 for stest in subst.get("subtests",[]): 554 for lfile in testDict[stest].get('localrunfiles','').split(): 555 install_files(os.path.join(root, lfile), 556 os.path.join(runscript_dir, os.path.dirname(lfile))) 557 558 # Now substitute the key variables into the header and footer 559 header=self._substVars(subst,example_template.header) 560 # The header is done twice to enable @...@ in header 561 header=self._substVars(subst,header) 562 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 563 564 # Start writing the file 565 fh.write(header+"\n") 566 567 # If there is a TODO or a SKIP then we do it before writing out the 568 # rest of the command (which is useful for working on the test) 569 # SKIP and TODO can be for the source file or for the runs 570 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 571 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 572 573 j=0 # for indentation 574 575 if loopVars: 576 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 577 if (loopHead): fh.write(loopHead+"\n") 578 579 # Subtests are special 580 allLoopVars=list(loopVars.keys()) 581 if 'subtests' in testDict: 582 substP=subst # Subtests can inherit args but be careful 583 k=0 # for label suffixes 584 for stest in testDict["subtests"]: 585 subst=substP.copy() 586 subst.update(testDict[stest]) 587 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 588 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 589 if sLoopVars: 590 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 591 allLoopVars+=list(sLoopVars.keys()) 592 fh.write(sLoopHead+"\n") 593 fh.write(self.getCmds(subst,j)+"\n") 594 if sLoopVars: 595 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 596 fh.write(sLoopFoot+"\n") 597 else: 598 fh.write(self.getCmds(subst,j)+"\n") 599 600 if loopVars: 601 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 602 fh.write(loopFoot+"\n") 603 604 fh.write(footer+"\n") 605 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 606 #if '10_9' in testname: sys.exit() 607 return 608 609 def genScriptsAndInfo(self,exfile,root,srcDict): 610 """ 611 Generate scripts from the source file, determine if built, etc. 612 For every test in the exfile with info in the srcDict: 613 1. Determine if it needs to be run for this arch 614 2. Generate the script 615 3. Generate the data needed to write out the makefile in a 616 convenient way 617 All tests are *always* run, but some may be SKIP'd per the TAP standard 618 """ 619 debug=False 620 rpath=self.srcrelpath(root) 621 execname=self.getExecname(exfile,rpath) 622 isBuilt=self._isBuilt(exfile,srcDict) 623 for test in srcDict: 624 if test in self.buildkeys: continue 625 if debug: print(self.nameSpace(exfile,root), test) 626 srcDict[test]['execname']=execname # Convenience in generating scripts 627 isRun=self._isRun(srcDict[test]) 628 self.genRunScript(test,root,isRun,srcDict) 629 srcDict[test]['isrun']=isRun 630 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 631 632 # This adds to datastructure for building deps 633 if isBuilt: self.addToSources(exfile,rpath,srcDict) 634 return 635 636 def _isBuilt(self,exfile,srcDict): 637 """ 638 Determine if this file should be built. 639 """ 640 # Get the language based on file extension 641 srcDict['SKIP'] = [] 642 lang=self.getLanguage(exfile) 643 if (lang=="F" or lang=="F90"): 644 if not self.have_fortran: 645 srcDict["SKIP"].append("Fortran required for this test") 646 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 647 srcDict["SKIP"].append("Fortran f90freeform required for this test") 648 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 649 srcDict["SKIP"].append("CUDA required for this test") 650 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 651 srcDict["SKIP"].append("C++ required for this test") 652 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 653 srcDict["SKIP"].append("C++ required for this test") 654 655 # Deprecated source files 656 if srcDict.get("TODO"): 657 return False 658 659 # isRun can work with srcDict to handle the requires 660 if "requires" in srcDict: 661 if srcDict["requires"]: 662 return self._isRun(srcDict) 663 664 return srcDict['SKIP'] == [] 665 666 667 def _isRun(self,testDict, debug=False): 668 """ 669 Based on the requirements listed in the src file and the petscconf.h 670 info, determine whether this test should be run or not. 671 """ 672 indent=" " 673 674 if 'SKIP' not in testDict: 675 testDict['SKIP'] = [] 676 # MPI requirements 677 if 'MPI_IS_MPIUNI' in self.conf: 678 if testDict.get('nsize', '1') != '1': 679 testDict['SKIP'].append("Parallel test with serial build") 680 681 # The requirements for the test are the sum of all the run subtests 682 if 'subtests' in testDict: 683 if 'requires' not in testDict: testDict['requires']="" 684 for stest in testDict['subtests']: 685 if 'requires' in testDict[stest]: 686 testDict['requires']+=" "+testDict[stest]['requires'] 687 if testDict.get('nsize', '1') != '1': 688 testDict['SKIP'].append("Parallel test with serial build") 689 break 690 691 # Now go through all requirements 692 if 'requires' in testDict: 693 for requirement in testDict['requires'].split(): 694 requirement=requirement.strip() 695 if not requirement: continue 696 if debug: print(indent+"Requirement: ", requirement) 697 isNull=False 698 if requirement.startswith("!"): 699 requirement=requirement[1:]; isNull=True 700 # Precision requirement for reals 701 if requirement in self.precision_types: 702 if self.conf['PETSC_PRECISION']==requirement: 703 if isNull: 704 testDict['SKIP'].append("not "+requirement+" required") 705 continue 706 continue # Success 707 elif not isNull: 708 testDict['SKIP'].append(requirement+" required") 709 continue 710 # Precision requirement for ints 711 if requirement in self.integer_types: 712 if requirement=="int32": 713 if self.conf['PETSC_SIZEOF_INT']==4: 714 if isNull: 715 testDict['SKIP'].append("not int32 required") 716 continue 717 continue # Success 718 elif not isNull: 719 testDict['SKIP'].append("int32 required") 720 continue 721 if requirement=="int64": 722 if self.conf['PETSC_SIZEOF_INT']==8: 723 if isNull: 724 testDict['SKIP'].append("NOT int64 required") 725 continue 726 continue # Success 727 elif not isNull: 728 testDict['SKIP'].append("int64 required") 729 continue 730 if requirement.startswith("long"): 731 reqsize = int(requirement[4:])//8 732 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 733 if longsize==reqsize: 734 if isNull: 735 testDict['SKIP'].append("not %s required" % requirement) 736 continue 737 continue # Success 738 elif not isNull: 739 testDict['SKIP'].append("%s required" % requirement) 740 continue 741 # Datafilespath 742 if requirement=="datafilespath" and not isNull: 743 testDict['SKIP'].append("Requires DATAFILESPATH") 744 continue 745 # Defines -- not sure I have comments matching 746 if "define(" in requirement.lower(): 747 reqdef=requirement.split("(")[1].split(")")[0] 748 if reqdef in self.conf: 749 if isNull: 750 testDict['SKIP'].append("Null requirement not met: "+requirement) 751 continue 752 continue # Success 753 elif not isNull: 754 testDict['SKIP'].append("Required: "+requirement) 755 continue 756 757 # Rest should be packages that we can just get from conf 758 if requirement == "complex": 759 petscconfvar="PETSC_USE_COMPLEX" 760 pkgconfvar="PETSC_USE_COMPLEX" 761 else: 762 petscconfvar="PETSC_HAVE_"+requirement.upper() 763 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 764 petsccv = self.conf.get(petscconfvar) 765 pkgcv = self.conf.get(pkgconfvar) 766 767 if petsccv or pkgcv: 768 if isNull: 769 if petsccv: 770 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 771 continue 772 else: 773 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 774 continue 775 continue # Success 776 elif not isNull: 777 if not petsccv and not pkgcv: 778 if debug: print("requirement not found: ", requirement) 779 if self.pkg_name == 'petsc': 780 testDict['SKIP'].append(petscconfvar+" requirement not met") 781 else: 782 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 783 continue 784 return testDict['SKIP'] == [] 785 786 def genPetscTests_summarize(self,dataDict): 787 """ 788 Required method to state what happened 789 """ 790 if not self.summarize: return 791 indent=" " 792 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 793 fh=open(fhname,"w") 794 for root in dataDict: 795 relroot=self.srcrelpath(root) 796 pkg=relroot.split("/")[1] 797 fh.write(relroot+"\n") 798 allSrcs=[] 799 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 800 for exfile in dataDict[root]: 801 # Basic information 802 rfile=os.path.join(relroot,exfile) 803 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 804 fh.write(indent+exfile+indent*4+builtStatus+"\n") 805 806 for test in dataDict[root][exfile]: 807 if test in self.buildkeys: continue 808 line=indent*2+test 809 fh.write(line+"\n") 810 # Looks nice to have the keys in order 811 #for key in dataDict[root][exfile][test]: 812 for key in "isrun abstracted nsize args requires script".split(): 813 if key not in dataDict[root][exfile][test]: continue 814 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 815 fh.write(line+"\n") 816 fh.write("\n") 817 fh.write("\n") 818 fh.write("\n") 819 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 820 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 821 fh.close() 822 return 823 824 def genPetscTests(self,root,dirs,files,dataDict): 825 """ 826 Go through and parse the source files in the directory to generate 827 the examples based on the metadata contained in the source files 828 """ 829 debug=False 830 # Use examplesAnalyze to get what the makefles think are sources 831 #self.examplesAnalyze(root,dirs,files,anlzDict) 832 833 dataDict[root]={} 834 835 for exfile in files: 836 #TST: Until we replace files, still leaving the orginals as is 837 #if not exfile.startswith("new_"+"ex"): continue 838 #if not exfile.startswith("ex"): continue 839 840 # Ignore emacs and other temporary files 841 if exfile.startswith("."): continue 842 if exfile.startswith("#"): continue 843 if exfile.endswith("~"): continue 844 # Only parse source files 845 ext=os.path.splitext(exfile)[-1].lstrip('.') 846 if ext not in LANGS: continue 847 848 # Convenience 849 fullex=os.path.join(root,exfile) 850 if self.verbose: print(' --> '+fullex) 851 dataDict[root].update(testparse.parseTestFile(fullex,0)) 852 if exfile in dataDict[root]: 853 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 854 855 return 856 857 def walktree(self,top): 858 """ 859 Walk a directory tree, starting from 'top' 860 """ 861 # Goal of action is to fill this dictionary 862 dataDict={} 863 for root, dirs, files in os.walk(top, topdown=True): 864 dirs.sort() 865 files.sort() 866 if not "examples" in root: continue 867 if "dSYM" in root: continue 868 if os.path.basename(root.rstrip("/")) == 'output': continue 869 if self.verbose: print(root) 870 self.genPetscTests(root,dirs,files,dataDict) 871 # Now summarize this dictionary 872 if self.verbose: self.genPetscTests_summarize(dataDict) 873 return dataDict 874 875 def gen_gnumake(self, fd): 876 """ 877 Overwrite of the method in the base PETSc class 878 """ 879 def write(stem, srcs): 880 for lang in LANGS: 881 if srcs[lang]['srcs']: 882 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 883 for pkg in self.pkg_pkgs: 884 srcs = self.gen_pkg(pkg) 885 write('testsrcs-' + pkg, srcs) 886 # Handle dependencies 887 for lang in LANGS: 888 for exfile in srcs[lang]['srcs']: 889 if exfile in srcs[lang]: 890 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 891 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 892 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 893 if deps: 894 # The executable literally depends on the object file because it is linked 895 fd.write(ex +": " + " ".join(deps) +'\n') 896 # The object file containing 'main' does not normally depend on other object 897 # files, but it does when it includes their modules. This dependency is 898 # overly blunt and could be reduced to only depend on object files for 899 # modules that are used, like "*f90aux.o". 900 fd.write(exfo +": " + " ".join(deps) +'\n') 901 902 return self.gendeps 903 904 def gen_pkg(self, pkg): 905 """ 906 Overwrite of the method in the base PETSc class 907 """ 908 return self.sources[pkg] 909 910 def write_gnumake(self, dataDict, output=None): 911 """ 912 Write out something similar to files from gmakegen.py 913 914 Test depends on script which also depends on source 915 file, but since I don't have a good way generating 916 acting on a single file (oops) just depend on 917 executable which in turn will depend on src file 918 """ 919 # Different options for how to set up the targets 920 compileExecsFirst=False 921 922 # Open file 923 fd = open(output, 'w') 924 925 # Write out the sources 926 gendeps = self.gen_gnumake(fd) 927 928 # Write out the tests and execname targets 929 fd.write("\n#Tests and executables\n") # Delimiter 930 931 for pkg in self.pkg_pkgs: 932 # These grab the ones that are built 933 for lang in LANGS: 934 testdeps=[] 935 for ftest in self.tests[pkg][lang]: 936 test=os.path.basename(ftest) 937 basedir=os.path.dirname(ftest) 938 testdeps.append(self.nameSpace(test,basedir)) 939 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 940 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 941 942 # test targets 943 for ftest in self.tests[pkg][lang]: 944 test=os.path.basename(ftest) 945 basedir=os.path.dirname(ftest) 946 testdir="${TESTDIR}/"+basedir+"/" 947 nmtest=self.nameSpace(test,basedir) 948 rundir=os.path.join(testdir,test) 949 script=test+".sh" 950 951 # Deps 952 exfile=self.tests[pkg][lang][ftest]['exfile'] 953 fullex=os.path.join(self.srcdir,exfile) 954 localexec=self.tests[pkg][lang][ftest]['exec'] 955 execname=os.path.join(testdir,localexec) 956 fullscript=os.path.join(testdir,script) 957 tmpfile=os.path.join(testdir,test,test+".tmp") 958 959 # *.counts depends on the script and either executable (will 960 # be run) or the example source file (SKIP or TODO) 961 fd.write('%s.counts : %s %s' 962 % (os.path.join('$(TESTDIR)/counts', nmtest), 963 fullscript, 964 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 965 ) 966 if exfile in self.sources[pkg][lang]: 967 for dep in self.sources[pkg][lang][exfile]: 968 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 969 fd.write('\n') 970 971 # Now write the args: 972 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 973 974 fd.close() 975 return 976 977def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 978 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 979 testdir=os.path.normpath(testdir) 980 if petsc_arch: 981 petsc_arch=petsc_arch.rstrip(os.path.sep) 982 if len(petsc_arch.split(os.path.sep))>1: 983 petsc_dir,petsc_arch=os.path.split(petsc_arch) 984 output = os.path.join(testdir, 'testfiles') 985 986 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 987 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 988 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 989 testdir=testdir) 990 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 991 pEx.write_gnumake(dataDict, output) 992 993if __name__ == '__main__': 994 import optparse 995 parser = optparse.OptionParser() 996 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 997 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 998 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 999 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1000 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1001 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1002 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1003 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1004 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1005 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1006 1007 opts, extra_args = parser.parse_args() 1008 if extra_args: 1009 import sys 1010 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1011 exit(1) 1012 if opts.testdir is None: 1013 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1014 1015 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1016 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1017 verbose=opts.verbose, 1018 single_ex=opts.single_executable, srcdir=opts.srcdir, 1019 testdir=opts.testdir) 1020