1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from collections import defaultdict 10from gmakegen import * 11 12import inspect 13thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 14sys.path.insert(0,thisscriptdir) 15import testparse 16import example_template 17 18 19""" 20 21There are 2 modes of running tests: Normal builds and run from prefix of 22install. They affect where to find things: 23 24 25Case 1. Normal builds: 26 27 +---------------------+----------------------------------+ 28 | PETSC_DIR | <git dir> | 29 +---------------------+----------------------------------+ 30 | PETSC_ARCH | arch-foo | 31 +---------------------+----------------------------------+ 32 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 33 +---------------------+----------------------------------+ 34 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 35 +---------------------+----------------------------------+ 36 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 37 +---------------------+----------------------------------+ 38 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 41 +---------------------+----------------------------------+ 42 43 44Case 2. From install dir: 45 46 +---------------------+-------------------------------------------------------+ 47 | PETSC_DIR | <prefix dir> | 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_ARCH | '' | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 60 +---------------------+-------------------------------------------------------+ 61 62""" 63 64def install_files(source, destdir): 65 """Install file or directory 'source' to 'destdir'. Does not preserve 66 mode (permissions). 67 """ 68 if not os.path.isdir(destdir): 69 os.makedirs(destdir) 70 if os.path.isdir(source): 71 for name in os.listdir(source): 72 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 73 else: 74 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 75 76class generateExamples(Petsc): 77 """ 78 gmakegen.py has basic structure for finding the files, writing out 79 the dependencies, etc. 80 """ 81 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 82 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 83 84 self.single_ex=single_ex 85 self.srcdir=srcdir 86 87 # Set locations to handle movement 88 self.inInstallDir=self.getInInstallDir(thisscriptdir) 89 90 if self.inInstallDir: 91 # Case 2 discussed above 92 # set PETSC_ARCH to install directory to allow script to work in both 93 dirlist=thisscriptdir.split(os.path.sep) 94 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 95 self.arch_dir=installdir 96 if self.srcdir is None: 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 if self.srcdir is None: 104 self.srcdir=os.path.join(self.petsc_dir,'src') 105 106 self.testroot_dir=os.path.abspath(testdir) 107 108 self.ptNaming=True 109 self.verbose=verbose 110 # Whether to write out a useful debugging 111 self.summarize=True if verbose else False 112 113 # For help in setting the requirements 114 self.precision_types="single double __float128 int32".split() 115 self.integer_types="int32 int64 long32 long64".split() 116 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 117 118 # Things that are not test 119 self.buildkeys=testparse.buildkeys 120 121 # Adding a dictionary for storing sources, objects, and tests 122 # to make building the dependency tree easier 123 self.sources={} 124 self.objects={} 125 self.tests={} 126 for pkg in self.pkg_pkgs: 127 self.sources[pkg]={} 128 self.objects[pkg]=[] 129 self.tests[pkg]={} 130 for lang in LANGS: 131 self.sources[pkg][lang]={} 132 self.sources[pkg][lang]['srcs']=[] 133 self.tests[pkg][lang]={} 134 135 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 136 137 self.indent=" " 138 if self.verbose: print('Finishing the constructor') 139 return 140 141 def srcrelpath(self,rdir): 142 """ 143 Get relative path to source directory 144 """ 145 return os.path.relpath(rdir,self.srcdir) 146 147 def getInInstallDir(self,thisscriptdir): 148 """ 149 When petsc is installed then this file in installed in: 150 <PREFIX>/share/petsc/examples/config/gmakegentest.py 151 otherwise the path is: 152 <PETSC_DIR>/config/gmakegentest.py 153 We use this difference to determine if we are in installdir 154 """ 155 dirlist=thisscriptdir.split(os.path.sep) 156 if len(dirlist)>4: 157 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 158 if lastfour==os.path.join('share','petsc','examples','config'): 159 return True 160 else: 161 return False 162 else: 163 return False 164 165 def nameSpace(self,srcfile,srcdir): 166 """ 167 Because the scripts have a non-unique naming, the pretty-printing 168 needs to convey the srcdir and srcfile. There are two ways of doing this. 169 """ 170 if self.ptNaming: 171 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 172 cdir=srcdir 173 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 174 nameString=prefix+srcfile 175 else: 176 #nameString=srcdir+": "+srcfile 177 nameString=srcfile 178 return nameString 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext=os.path.splitext(srcfile)[-1] 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".cpp".split(): langReq="cpp" 191 if srcext == ".cu": langReq="cu" 192 if srcext == ".c": langReq="c" 193 #if not langReq: print("ERROR: ", srcext, srcfile) 194 return langReq 195 196 def _getLoopVars(self,inDict,testname, isSubtest=False): 197 """ 198 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 199 Return: 200 inDict['args']: -ksp_monitor 201 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 202 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 203 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 204 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 205 subst should be passed in instead of inDict 206 """ 207 loopVars={}; newargs=[] 208 lsuffix='_' 209 argregex = re.compile(' (?=-[a-zA-Z])') 210 from testparse import parseLoopArgs 211 for key in inDict: 212 if key in ('SKIP', 'regexes'): 213 continue 214 akey=('subargs' if key=='args' else key) # what to assign 215 if akey not in inDict: inDict[akey]='' 216 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 217 # Always generate a loop over nsize, even if there is only one value 218 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 219 keystr = str(inDict[key]) 220 varlist = [] 221 for varset in argregex.split(keystr): 222 if not varset.strip(): continue 223 if '{{' in varset: 224 keyvar,lvars,ftype=parseLoopArgs(varset) 225 if akey not in loopVars: loopVars[akey]={} 226 varlist.append(keyvar) 227 loopVars[akey][keyvar]=[keyvar,lvars] 228 if akey=='nsize': 229 if len(lvars.split()) > 1: 230 lsuffix += akey +'-${' + keyvar + '}' 231 else: 232 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 233 lsuffix+=keyvar+'-${' + keyvar + '}_' 234 else: 235 if key=='args': 236 newargs.append(varset.strip()) 237 if varlist: 238 loopVars[akey]['varlist']=varlist 239 240 # For subtests, args are always substituted in (not top level) 241 if isSubtest: 242 inDict['subargs'] += " ".join(newargs) 243 inDict['args']='' 244 if 'label_suffix' in inDict: 245 inDict['label_suffix']+=lsuffix.rstrip('_') 246 else: 247 inDict['label_suffix']=lsuffix.rstrip('_') 248 else: 249 if loopVars: 250 inDict['args'] = ' '.join(newargs) 251 inDict['label_suffix']=lsuffix.rstrip('_') 252 return loopVars 253 254 def getArgLabel(self,testDict): 255 """ 256 In all of the arguments in the test dictionary, create a simple 257 string for searching within the makefile system. For simplicity in 258 search, remove "-", for strings, etc. 259 Also, concatenate the arg commands 260 For now, ignore nsize -- seems hard to search for anyway 261 """ 262 # Collect all of the args associated with a test 263 argStr=("" if 'args' not in testDict else testDict['args']) 264 if 'subtests' in testDict: 265 for stest in testDict["subtests"]: 266 sd=testDict[stest] 267 argStr=argStr+("" if 'args' not in sd else sd['args']) 268 269 # Now go through and cleanup 270 argStr=re.sub('{{(.*?)}}',"",argStr) 271 argStr=re.sub('-'," ",argStr) 272 for digit in string.digits: argStr=re.sub(digit," ",argStr) 273 argStr=re.sub("\.","",argStr) 274 argStr=re.sub(",","",argStr) 275 argStr=re.sub('\+',' ',argStr) 276 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 277 return argStr.strip() 278 279 def addToSources(self,exfile,rpath,srcDict): 280 """ 281 Put into data structure that allows easy generation of makefile 282 """ 283 pkg=rpath.split(os.path.sep)[0] 284 relpfile=os.path.join(rpath,exfile) 285 lang=self.getLanguage(exfile) 286 if not lang: return 287 if pkg not in self.sources: return 288 self.sources[pkg][lang]['srcs'].append(relpfile) 289 self.sources[pkg][lang][relpfile] = [] 290 if 'depends' in srcDict: 291 depSrcList=srcDict['depends'].split() 292 for depSrc in depSrcList: 293 depObj=os.path.splitext(depSrc)[0]+".o" 294 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 295 296 # In gmakefile, ${TESTDIR} var specifies the object compilation 297 testsdir=rpath+"/" 298 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 299 self.objects[pkg].append(objfile) 300 return 301 302 def addToTests(self,test,rpath,exfile,execname,testDict): 303 """ 304 Put into data structure that allows easy generation of makefile 305 Organized by languages to allow testing of languages 306 """ 307 pkg=rpath.split("/")[0] 308 nmtest=os.path.join(rpath,test) 309 lang=self.getLanguage(exfile) 310 if not lang: return 311 if pkg not in self.tests: return 312 self.tests[pkg][lang][nmtest]={} 313 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 314 self.tests[pkg][lang][nmtest]['exec']=execname 315 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 316 return 317 318 def getExecname(self,exfile,rpath): 319 """ 320 Generate bash script using template found next to this file. 321 This file is read in at constructor time to avoid file I/O 322 """ 323 if self.single_ex: 324 execname=rpath.split("/")[1]+"-ex" 325 else: 326 execname=os.path.splitext(exfile)[0] 327 return execname 328 329 def getSubstVars(self,testDict,rpath,testname): 330 """ 331 Create a dictionary with all of the variables that get substituted 332 into the template commands found in example_template.py 333 """ 334 subst={} 335 336 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 337 if 'nsize' not in testDict: testDict['nsize'] = '1' 338 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 339 for ak in testparse.acceptedkeys: 340 if ak=='test': continue 341 subst[ak]=(testDict[ak] if ak in testDict else '') 342 343 # Now do other variables 344 subst['execname']=testDict['execname'] 345 if 'filter' in testDict: 346 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 347 348 # Others 349 subst['subargs']='' # Default. For variables override 350 subst['srcdir']=os.path.join(self.srcdir, rpath) 351 subst['label_suffix']='' 352 subst['comments']="\n#".join(subst['comments'].split("\n")) 353 if subst['comments']: subst['comments']="#"+subst['comments'] 354 subst['exec']="../"+subst['execname'] 355 subst['testroot']=self.testroot_dir 356 subst['testname']=testname 357 dp = self.conf.get('DATAFILESPATH','') 358 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 359 360 # This is used to label some matrices 361 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 362 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 363 364 #Conf vars 365 if self.petsc_arch.find('valgrind')>=0: 366 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 367 else: 368 subst['mpiexec']=self.conf['MPIEXEC'] 369 subst['pkg_name']=self.pkg_name 370 subst['pkg_dir']=self.pkg_dir 371 subst['pkg_arch']=self.petsc_arch 372 subst['CONFIG_DIR']=thisscriptdir 373 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 374 subst['diff']=self.conf['DIFF'] 375 subst['rm']=self.conf['RM'] 376 subst['grep']=self.conf['GREP'] 377 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 378 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 379 380 # Output file is special because of subtests override 381 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 382 if not "_" in defroot: defroot=defroot+"_1" 383 subst['defroot']=defroot 384 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 385 subst['redirect_file']=defroot+".tmp" 386 if 'output_file' not in testDict: 387 subst['output_file']="output/"+defroot+".out" 388 # Add in the full path here. 389 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 390 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 391 if not subst['TODO']: 392 print("Warning: "+subst['output_file']+" not found.") 393 # Worry about alt files here -- see 394 # src/snes/examples/tutorials/output/ex22*.out 395 altlist=[subst['output_file']] 396 basefile,ext = os.path.splitext(subst['output_file']) 397 for i in range(1,9): 398 altroot=basefile+"_alt" 399 if i > 1: altroot=altroot+"_"+str(i) 400 af=altroot+".out" 401 srcaf=os.path.join(subst['srcdir'],af) 402 fullaf=os.path.join(self.petsc_dir,srcaf) 403 if os.path.isfile(fullaf): altlist.append(srcaf) 404 if len(altlist)>1: subst['altfiles']=altlist 405 #if len(altlist)>1: print("Found alt files: ",altlist) 406 407 subst['regexes']={} 408 for subkey in subst: 409 if subkey=='regexes': continue 410 if not isinstance(subst[subkey],str): continue 411 patt="@"+subkey.upper()+"@" 412 subst['regexes'][subkey]=re.compile(patt) 413 414 return subst 415 416 def _substVars(self,subst,origStr): 417 """ 418 Substitute variables 419 """ 420 Str=origStr 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 if subkey.upper() not in Str: continue 425 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 426 return Str 427 428 def getCmds(self,subst,i): 429 """ 430 Generate bash script using template found next to this file. 431 This file is read in at constructor time to avoid file I/O 432 """ 433 nindnt=i # the start and has to be consistent with below 434 cmdindnt=self.indent*nindnt 435 cmdLines="" 436 437 # MPI is the default -- but we have a few odd commands 438 if not subst['command']: 439 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 440 else: 441 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 442 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 443 444 cmdLines+=cmdindnt+'if test $res = 0; then\n' 445 diffindnt=self.indent*(nindnt+1) 446 if not subst['filter_output']: 447 if 'altfiles' not in subst: 448 cmd=diffindnt+self._substVars(subst,example_template.difftest) 449 else: 450 # Have to do it by hand a bit because of variable number of alt files 451 rf=subst['redirect_file'] 452 cmd=diffindnt+example_template.difftest.split('@')[0] 453 for i in range(len(subst['altfiles'])): 454 af=subst['altfiles'][i] 455 cmd+=af+' '+rf 456 if i!=len(subst['altfiles'])-1: 457 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 458 cmd+=' || ${diff_exe} ' 459 else: 460 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 461 cmd+=subst['label_suffix']+' ""' # Quotes are painful 462 else: 463 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 464 cmdLines+=cmd+"\n" 465 cmdLines+=cmdindnt+'else\n' 466 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 467 cmdLines+=cmdindnt+'fi\n' 468 return cmdLines 469 470 def _writeTodoSkip(self,fh,tors,reasons,footer): 471 """ 472 Write out the TODO and SKIP lines in the file 473 The TODO or SKIP variable, tors, should be lower case 474 """ 475 TORS=tors.upper() 476 template=eval("example_template."+tors+"line") 477 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 478 tab = '' 479 if reasons: 480 fh.write('if ! $force; then\n') 481 tab = tab + ' ' 482 if reasons == ["Requires DATAFILESPATH"]: 483 # The only reason not to run is DATAFILESPATH, which we check at run-time 484 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 485 tab = tab + ' ' 486 if reasons: 487 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 488 fh.write(tab+footer+"\n") 489 fh.write(tab+"exit\n") 490 if reasons == ["Requires DATAFILESPATH"]: 491 fh.write(' fi\n') 492 if reasons: 493 fh.write('fi\n') 494 fh.write('\n\n') 495 return 496 497 def getLoopVarsHead(self,loopVars,i,usedVars={}): 498 """ 499 Generate a nicely indented string with the format loops 500 Here is what the data structure looks like 501 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 502 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 503 loopVars['subargs']['pc_type']=["j","cholesky sor"] 504 """ 505 outstr=''; indnt=self.indent 506 507 for key in loopVars: 508 if key in usedVars: continue # Do not duplicate setting vars 509 for var in loopVars[key]['varlist']: 510 varval=loopVars[key][var] 511 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 512 outstr += "\n\n" 513 514 for key in loopVars: 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 518 i = i + 1 519 return (outstr,i) 520 521 def getLoopVarsFoot(self,loopVars,i): 522 outstr=''; indnt=self.indent 523 for key in loopVars: 524 for var in loopVars[key]['varlist']: 525 i = i - 1 526 outstr += indnt * i + "done\n" 527 return (outstr,i) 528 529 def genRunScript(self,testname,root,isRun,srcDict): 530 """ 531 Generate bash script using template found next to this file. 532 This file is read in at constructor time to avoid file I/O 533 """ 534 # runscript_dir directory has to be consistent with gmakefile 535 testDict=srcDict[testname] 536 rpath=self.srcrelpath(root) 537 runscript_dir=os.path.join(self.testroot_dir,rpath) 538 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 539 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 540 541 # Get variables to go into shell scripts. last time testDict used 542 subst=self.getSubstVars(testDict,rpath,testname) 543 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 544 if 'subtests' in testDict: 545 # The subtests inherit inDict, so we don't need top-level loops. 546 loopVars = {} 547 548 #Handle runfiles 549 for lfile in subst.get('localrunfiles','').split(): 550 install_files(os.path.join(root, lfile), 551 os.path.join(runscript_dir, os.path.dirname(lfile))) 552 # Check subtests for local runfiles 553 for stest in subst.get("subtests",[]): 554 for lfile in testDict[stest].get('localrunfiles','').split(): 555 install_files(os.path.join(root, lfile), 556 os.path.join(runscript_dir, os.path.dirname(lfile))) 557 558 # Now substitute the key variables into the header and footer 559 header=self._substVars(subst,example_template.header) 560 # The header is done twice to enable @...@ in header 561 header=self._substVars(subst,header) 562 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 563 564 # Start writing the file 565 fh.write(header+"\n") 566 567 # If there is a TODO or a SKIP then we do it before writing out the 568 # rest of the command (which is useful for working on the test) 569 # SKIP and TODO can be for the source file or for the runs 570 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 571 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 572 573 j=0 # for indentation 574 575 if loopVars: 576 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 577 if (loopHead): fh.write(loopHead+"\n") 578 579 # Subtests are special 580 allLoopVars=list(loopVars.keys()) 581 if 'subtests' in testDict: 582 substP=subst # Subtests can inherit args but be careful 583 k=0 # for label suffixes 584 for stest in testDict["subtests"]: 585 subst=substP.copy() 586 subst.update(testDict[stest]) 587 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 588 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 589 if sLoopVars: 590 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 591 allLoopVars+=list(sLoopVars.keys()) 592 fh.write(sLoopHead+"\n") 593 fh.write(self.getCmds(subst,j)+"\n") 594 if sLoopVars: 595 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 596 fh.write(sLoopFoot+"\n") 597 else: 598 fh.write(self.getCmds(subst,j)+"\n") 599 600 if loopVars: 601 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 602 fh.write(loopFoot+"\n") 603 604 fh.write(footer+"\n") 605 fh.close() 606 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 607 #if '10_9' in testname: sys.exit() 608 return 609 610 def genScriptsAndInfo(self,exfile,root,srcDict): 611 """ 612 Generate scripts from the source file, determine if built, etc. 613 For every test in the exfile with info in the srcDict: 614 1. Determine if it needs to be run for this arch 615 2. Generate the script 616 3. Generate the data needed to write out the makefile in a 617 convenient way 618 All tests are *always* run, but some may be SKIP'd per the TAP standard 619 """ 620 debug=False 621 rpath=self.srcrelpath(root) 622 execname=self.getExecname(exfile,rpath) 623 isBuilt=self._isBuilt(exfile,srcDict) 624 for test in srcDict: 625 if test in self.buildkeys: continue 626 if debug: print(self.nameSpace(exfile,root), test) 627 srcDict[test]['execname']=execname # Convenience in generating scripts 628 isRun=self._isRun(srcDict[test]) 629 self.genRunScript(test,root,isRun,srcDict) 630 srcDict[test]['isrun']=isRun 631 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 632 633 # This adds to datastructure for building deps 634 if isBuilt: self.addToSources(exfile,rpath,srcDict) 635 return 636 637 def _isBuilt(self,exfile,srcDict): 638 """ 639 Determine if this file should be built. 640 """ 641 # Get the language based on file extension 642 srcDict['SKIP'] = [] 643 lang=self.getLanguage(exfile) 644 if (lang=="F" or lang=="F90"): 645 if not self.have_fortran: 646 srcDict["SKIP"].append("Fortran required for this test") 647 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 648 srcDict["SKIP"].append("Fortran f90freeform required for this test") 649 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 650 srcDict["SKIP"].append("CUDA required for this test") 651 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 652 srcDict["SKIP"].append("C++ required for this test") 653 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 654 srcDict["SKIP"].append("C++ required for this test") 655 656 # Deprecated source files 657 if srcDict.get("TODO"): 658 return False 659 660 # isRun can work with srcDict to handle the requires 661 if "requires" in srcDict: 662 if srcDict["requires"]: 663 return self._isRun(srcDict) 664 665 return srcDict['SKIP'] == [] 666 667 668 def _isRun(self,testDict, debug=False): 669 """ 670 Based on the requirements listed in the src file and the petscconf.h 671 info, determine whether this test should be run or not. 672 """ 673 indent=" " 674 675 if 'SKIP' not in testDict: 676 testDict['SKIP'] = [] 677 # MPI requirements 678 if 'MPI_IS_MPIUNI' in self.conf: 679 if testDict.get('nsize', '1') != '1': 680 testDict['SKIP'].append("Parallel test with serial build") 681 682 # The requirements for the test are the sum of all the run subtests 683 if 'subtests' in testDict: 684 if 'requires' not in testDict: testDict['requires']="" 685 for stest in testDict['subtests']: 686 if 'requires' in testDict[stest]: 687 testDict['requires']+=" "+testDict[stest]['requires'] 688 if testDict.get('nsize', '1') != '1': 689 testDict['SKIP'].append("Parallel test with serial build") 690 break 691 692 # Now go through all requirements 693 if 'requires' in testDict: 694 for requirement in testDict['requires'].split(): 695 requirement=requirement.strip() 696 if not requirement: continue 697 if debug: print(indent+"Requirement: ", requirement) 698 isNull=False 699 if requirement.startswith("!"): 700 requirement=requirement[1:]; isNull=True 701 # Precision requirement for reals 702 if requirement in self.precision_types: 703 if self.conf['PETSC_PRECISION']==requirement: 704 if isNull: 705 testDict['SKIP'].append("not "+requirement+" required") 706 continue 707 continue # Success 708 elif not isNull: 709 testDict['SKIP'].append(requirement+" required") 710 continue 711 # Precision requirement for ints 712 if requirement in self.integer_types: 713 if requirement=="int32": 714 if self.conf['PETSC_SIZEOF_INT']==4: 715 if isNull: 716 testDict['SKIP'].append("not int32 required") 717 continue 718 continue # Success 719 elif not isNull: 720 testDict['SKIP'].append("int32 required") 721 continue 722 if requirement=="int64": 723 if self.conf['PETSC_SIZEOF_INT']==8: 724 if isNull: 725 testDict['SKIP'].append("NOT int64 required") 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append("int64 required") 730 continue 731 if requirement.startswith("long"): 732 reqsize = int(requirement[4:])//8 733 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 734 if longsize==reqsize: 735 if isNull: 736 testDict['SKIP'].append("not %s required" % requirement) 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("%s required" % requirement) 741 continue 742 # Datafilespath 743 if requirement=="datafilespath" and not isNull: 744 testDict['SKIP'].append("Requires DATAFILESPATH") 745 continue 746 # Defines -- not sure I have comments matching 747 if "define(" in requirement.lower(): 748 reqdef=requirement.split("(")[1].split(")")[0] 749 if reqdef in self.conf: 750 if isNull: 751 testDict['SKIP'].append("Null requirement not met: "+requirement) 752 continue 753 continue # Success 754 elif not isNull: 755 testDict['SKIP'].append("Required: "+requirement) 756 continue 757 758 # Rest should be packages that we can just get from conf 759 if requirement in ["complex","debug"]: 760 petscconfvar="PETSC_USE_"+requirement.upper() 761 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 762 else: 763 petscconfvar="PETSC_HAVE_"+requirement.upper() 764 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 765 petsccv = self.conf.get(petscconfvar) 766 pkgcv = self.conf.get(pkgconfvar) 767 768 if petsccv or pkgcv: 769 if isNull: 770 if petsccv: 771 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 772 continue 773 else: 774 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 775 continue 776 continue # Success 777 elif not isNull: 778 if not petsccv and not pkgcv: 779 if debug: print("requirement not found: ", requirement) 780 if self.pkg_name == 'petsc': 781 testDict['SKIP'].append(petscconfvar+" requirement not met") 782 else: 783 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 784 continue 785 return testDict['SKIP'] == [] 786 787 def genPetscTests_summarize(self,dataDict): 788 """ 789 Required method to state what happened 790 """ 791 if not self.summarize: return 792 indent=" " 793 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 794 fh=open(fhname,"w") 795 for root in dataDict: 796 relroot=self.srcrelpath(root) 797 pkg=relroot.split("/")[1] 798 fh.write(relroot+"\n") 799 allSrcs=[] 800 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 801 for exfile in dataDict[root]: 802 # Basic information 803 rfile=os.path.join(relroot,exfile) 804 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 805 fh.write(indent+exfile+indent*4+builtStatus+"\n") 806 807 for test in dataDict[root][exfile]: 808 if test in self.buildkeys: continue 809 line=indent*2+test 810 fh.write(line+"\n") 811 # Looks nice to have the keys in order 812 #for key in dataDict[root][exfile][test]: 813 for key in "isrun abstracted nsize args requires script".split(): 814 if key not in dataDict[root][exfile][test]: continue 815 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 816 fh.write(line+"\n") 817 fh.write("\n") 818 fh.write("\n") 819 fh.write("\n") 820 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 821 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 822 fh.close() 823 return 824 825 def genPetscTests(self,root,dirs,files,dataDict): 826 """ 827 Go through and parse the source files in the directory to generate 828 the examples based on the metadata contained in the source files 829 """ 830 debug=False 831 # Use examplesAnalyze to get what the makefles think are sources 832 #self.examplesAnalyze(root,dirs,files,anlzDict) 833 834 dataDict[root]={} 835 836 for exfile in files: 837 #TST: Until we replace files, still leaving the orginals as is 838 #if not exfile.startswith("new_"+"ex"): continue 839 #if not exfile.startswith("ex"): continue 840 841 # Ignore emacs and other temporary files 842 if exfile.startswith("."): continue 843 if exfile.startswith("#"): continue 844 if exfile.endswith("~"): continue 845 # Only parse source files 846 ext=os.path.splitext(exfile)[-1].lstrip('.') 847 if ext not in LANGS: continue 848 849 # Convenience 850 fullex=os.path.join(root,exfile) 851 if self.verbose: print(' --> '+fullex) 852 dataDict[root].update(testparse.parseTestFile(fullex,0)) 853 if exfile in dataDict[root]: 854 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 855 856 return 857 858 def walktree(self,top): 859 """ 860 Walk a directory tree, starting from 'top' 861 """ 862 # Goal of action is to fill this dictionary 863 dataDict={} 864 for root, dirs, files in os.walk(top, topdown=True): 865 dirs.sort() 866 files.sort() 867 if not "examples" in root: continue 868 if "dSYM" in root: continue 869 if os.path.basename(root.rstrip("/")) == 'output': continue 870 if self.verbose: print(root) 871 self.genPetscTests(root,dirs,files,dataDict) 872 # Now summarize this dictionary 873 if self.verbose: self.genPetscTests_summarize(dataDict) 874 return dataDict 875 876 def gen_gnumake(self, fd): 877 """ 878 Overwrite of the method in the base PETSc class 879 """ 880 def write(stem, srcs): 881 for lang in LANGS: 882 if srcs[lang]['srcs']: 883 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 884 for pkg in self.pkg_pkgs: 885 srcs = self.gen_pkg(pkg) 886 write('testsrcs-' + pkg, srcs) 887 # Handle dependencies 888 for lang in LANGS: 889 for exfile in srcs[lang]['srcs']: 890 if exfile in srcs[lang]: 891 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 892 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 893 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 894 if deps: 895 # The executable literally depends on the object file because it is linked 896 fd.write(ex +": " + " ".join(deps) +'\n') 897 # The object file containing 'main' does not normally depend on other object 898 # files, but it does when it includes their modules. This dependency is 899 # overly blunt and could be reduced to only depend on object files for 900 # modules that are used, like "*f90aux.o". 901 fd.write(exfo +": " + " ".join(deps) +'\n') 902 903 return self.gendeps 904 905 def gen_pkg(self, pkg): 906 """ 907 Overwrite of the method in the base PETSc class 908 """ 909 return self.sources[pkg] 910 911 def write_gnumake(self, dataDict, output=None): 912 """ 913 Write out something similar to files from gmakegen.py 914 915 Test depends on script which also depends on source 916 file, but since I don't have a good way generating 917 acting on a single file (oops) just depend on 918 executable which in turn will depend on src file 919 """ 920 # Different options for how to set up the targets 921 compileExecsFirst=False 922 923 # Open file 924 fd = open(output, 'w') 925 926 # Write out the sources 927 gendeps = self.gen_gnumake(fd) 928 929 # Write out the tests and execname targets 930 fd.write("\n#Tests and executables\n") # Delimiter 931 932 for pkg in self.pkg_pkgs: 933 # These grab the ones that are built 934 for lang in LANGS: 935 testdeps=[] 936 for ftest in self.tests[pkg][lang]: 937 test=os.path.basename(ftest) 938 basedir=os.path.dirname(ftest) 939 testdeps.append(self.nameSpace(test,basedir)) 940 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 941 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 942 943 # test targets 944 for ftest in self.tests[pkg][lang]: 945 test=os.path.basename(ftest) 946 basedir=os.path.dirname(ftest) 947 testdir="${TESTDIR}/"+basedir+"/" 948 nmtest=self.nameSpace(test,basedir) 949 rundir=os.path.join(testdir,test) 950 script=test+".sh" 951 952 # Deps 953 exfile=self.tests[pkg][lang][ftest]['exfile'] 954 fullex=os.path.join(self.srcdir,exfile) 955 localexec=self.tests[pkg][lang][ftest]['exec'] 956 execname=os.path.join(testdir,localexec) 957 fullscript=os.path.join(testdir,script) 958 tmpfile=os.path.join(testdir,test,test+".tmp") 959 960 # *.counts depends on the script and either executable (will 961 # be run) or the example source file (SKIP or TODO) 962 fd.write('%s.counts : %s %s' 963 % (os.path.join('$(TESTDIR)/counts', nmtest), 964 fullscript, 965 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 966 ) 967 if exfile in self.sources[pkg][lang]: 968 for dep in self.sources[pkg][lang][exfile]: 969 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 970 fd.write('\n') 971 972 # Now write the args: 973 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 974 975 fd.close() 976 return 977 978def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 979 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 980 testdir=os.path.normpath(testdir) 981 if petsc_arch: 982 petsc_arch=petsc_arch.rstrip(os.path.sep) 983 if len(petsc_arch.split(os.path.sep))>1: 984 petsc_dir,petsc_arch=os.path.split(petsc_arch) 985 output = os.path.join(testdir, 'testfiles') 986 987 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 988 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 989 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 990 testdir=testdir) 991 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 992 pEx.write_gnumake(dataDict, output) 993 994if __name__ == '__main__': 995 import optparse 996 parser = optparse.OptionParser() 997 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 998 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 999 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1000 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1001 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1002 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1003 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1004 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1005 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1006 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1007 1008 opts, extra_args = parser.parse_args() 1009 if extra_args: 1010 import sys 1011 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1012 exit(1) 1013 if opts.testdir is None: 1014 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1015 1016 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1017 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1018 verbose=opts.verbose, 1019 single_ex=opts.single_executable, srcdir=opts.srcdir, 1020 testdir=opts.testdir) 1021