1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 84 85 self.single_ex=single_ex 86 self.srcdir=srcdir 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 if self.srcdir is None: 98 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 99 else: 100 if petsc_arch == '': 101 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 102 # Case 1 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 if self.srcdir is None: 105 self.srcdir=os.path.join(self.petsc_dir,'src') 106 107 self.testroot_dir=os.path.abspath(testdir) 108 109 self.ptNaming=True 110 self.verbose=verbose 111 # Whether to write out a useful debugging 112 self.summarize=True if verbose else False 113 114 # For help in setting the requirements 115 self.precision_types="single double __float128 int32".split() 116 self.integer_types="int32 int64 long32 long64".split() 117 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 118 119 # Things that are not test 120 self.buildkeys=testparse.buildkeys 121 122 # Adding a dictionary for storing sources, objects, and tests 123 # to make building the dependency tree easier 124 self.sources={} 125 self.objects={} 126 self.tests={} 127 for pkg in self.pkg_pkgs: 128 self.sources[pkg]={} 129 self.objects[pkg]=[] 130 self.tests[pkg]={} 131 for lang in LANGS: 132 self.sources[pkg][lang]={} 133 self.sources[pkg][lang]['srcs']=[] 134 self.tests[pkg][lang]={} 135 136 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 137 138 self.indent=" " 139 if self.verbose: print('Finishing the constructor') 140 return 141 142 def srcrelpath(self,rdir): 143 """ 144 Get relative path to source directory 145 """ 146 return os.path.relpath(rdir,self.srcdir) 147 148 def getInInstallDir(self,thisscriptdir): 149 """ 150 When petsc is installed then this file in installed in: 151 <PREFIX>/share/petsc/examples/config/gmakegentest.py 152 otherwise the path is: 153 <PETSC_DIR>/config/gmakegentest.py 154 We use this difference to determine if we are in installdir 155 """ 156 dirlist=thisscriptdir.split(os.path.sep) 157 if len(dirlist)>4: 158 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 159 if lastfour==os.path.join('share','petsc','examples','config'): 160 return True 161 else: 162 return False 163 else: 164 return False 165 166 def nameSpace(self,srcfile,srcdir): 167 """ 168 Because the scripts have a non-unique naming, the pretty-printing 169 needs to convey the srcdir and srcfile. There are two ways of doing this. 170 """ 171 if self.ptNaming: 172 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 173 cdir=srcdir 174 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 175 nameString=prefix+srcfile 176 else: 177 #nameString=srcdir+": "+srcfile 178 nameString=srcfile 179 return nameString 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:\] now? 185 """ 186 langReq=None 187 srcext=os.path.splitext(srcfile)[-1] 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 if pkg not in self.sources: return 289 self.sources[pkg][lang]['srcs'].append(relpfile) 290 self.sources[pkg][lang][relpfile] = [] 291 if 'depends' in srcDict: 292 depSrcList=srcDict['depends'].split() 293 for depSrc in depSrcList: 294 depObj=os.path.splitext(depSrc)[0]+".o" 295 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 296 297 # In gmakefile, ${TESTDIR} var specifies the object compilation 298 testsdir=rpath+"/" 299 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 300 self.objects[pkg].append(objfile) 301 return 302 303 def addToTests(self,test,rpath,exfile,execname,testDict): 304 """ 305 Put into data structure that allows easy generation of makefile 306 Organized by languages to allow testing of languages 307 """ 308 pkg=rpath.split("/")[0] 309 nmtest=os.path.join(rpath,test) 310 lang=self.getLanguage(exfile) 311 if not lang: return 312 if pkg not in self.tests: return 313 self.tests[pkg][lang][nmtest]={} 314 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 315 self.tests[pkg][lang][nmtest]['exec']=execname 316 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 317 return 318 319 def getExecname(self,exfile,rpath): 320 """ 321 Generate bash script using template found next to this file. 322 This file is read in at constructor time to avoid file I/O 323 """ 324 if self.single_ex: 325 execname=rpath.split("/")[1]+"-ex" 326 else: 327 execname=os.path.splitext(exfile)[0] 328 return execname 329 330 def getSubstVars(self,testDict,rpath,testname): 331 """ 332 Create a dictionary with all of the variables that get substituted 333 into the template commands found in example_template.py 334 """ 335 subst={} 336 337 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 338 if 'nsize' not in testDict: testDict['nsize'] = '1' 339 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 340 for ak in testparse.acceptedkeys: 341 if ak=='test': continue 342 subst[ak]=(testDict[ak] if ak in testDict else '') 343 344 # Now do other variables 345 subst['execname']=testDict['execname'] 346 if 'filter' in testDict: 347 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(self.srcdir, rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['pkg_name']=self.pkg_name 371 subst['pkg_dir']=self.pkg_dir 372 subst['pkg_arch']=self.petsc_arch 373 subst['CONFIG_DIR']=thisscriptdir 374 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 375 subst['diff']=self.conf['DIFF'] 376 subst['rm']=self.conf['RM'] 377 subst['grep']=self.conf['GREP'] 378 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 379 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 380 381 # Output file is special because of subtests override 382 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 383 if not "_" in defroot: defroot=defroot+"_1" 384 subst['defroot']=defroot 385 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 386 subst['redirect_file']=defroot+".tmp" 387 if 'output_file' not in testDict: 388 subst['output_file']="output/"+defroot+".out" 389 # Add in the full path here. 390 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 391 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 392 if not subst['TODO']: 393 print("Warning: "+subst['output_file']+" not found.") 394 # Worry about alt files here -- see 395 # src/snes/examples/tutorials/output/ex22*.out 396 altlist=[subst['output_file']] 397 basefile,ext = os.path.splitext(subst['output_file']) 398 for i in range(1,9): 399 altroot=basefile+"_alt" 400 if i > 1: altroot=altroot+"_"+str(i) 401 af=altroot+".out" 402 srcaf=os.path.join(subst['srcdir'],af) 403 fullaf=os.path.join(self.petsc_dir,srcaf) 404 if os.path.isfile(fullaf): altlist.append(srcaf) 405 if len(altlist)>1: subst['altfiles']=altlist 406 #if len(altlist)>1: print("Found alt files: ",altlist) 407 408 subst['regexes']={} 409 for subkey in subst: 410 if subkey=='regexes': continue 411 if not isinstance(subst[subkey],str): continue 412 patt="@"+subkey.upper()+"@" 413 subst['regexes'][subkey]=re.compile(patt) 414 415 return subst 416 417 def _substVars(self,subst,origStr): 418 """ 419 Substitute variables 420 """ 421 Str=origStr 422 for subkey in subst: 423 if subkey=='regexes': continue 424 if not isinstance(subst[subkey],str): continue 425 if subkey.upper() not in Str: continue 426 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 427 return Str 428 429 def getCmds(self,subst,i): 430 """ 431 Generate bash script using template found next to this file. 432 This file is read in at constructor time to avoid file I/O 433 """ 434 nindnt=i # the start and has to be consistent with below 435 cmdindnt=self.indent*nindnt 436 cmdLines="" 437 438 # MPI is the default -- but we have a few odd commands 439 if not subst['command']: 440 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 441 else: 442 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 443 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 444 445 cmdLines+=cmdindnt+'if test $res = 0; then\n' 446 diffindnt=self.indent*(nindnt+1) 447 if not subst['filter_output']: 448 if 'altfiles' not in subst: 449 cmd=diffindnt+self._substVars(subst,example_template.difftest) 450 else: 451 # Have to do it by hand a bit because of variable number of alt files 452 rf=subst['redirect_file'] 453 cmd=diffindnt+example_template.difftest.split('@')[0] 454 for i in range(len(subst['altfiles'])): 455 af=subst['altfiles'][i] 456 cmd+=af+' '+rf 457 if i!=len(subst['altfiles'])-1: 458 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 459 cmd+=' || ${diff_exe} ' 460 else: 461 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 462 cmd+=subst['label_suffix']+' ""' # Quotes are painful 463 else: 464 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 465 cmdLines+=cmd+"\n" 466 cmdLines+=cmdindnt+'else\n' 467 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 468 cmdLines+=cmdindnt+'fi\n' 469 return cmdLines 470 471 def _writeTodoSkip(self,fh,tors,reasons,footer): 472 """ 473 Write out the TODO and SKIP lines in the file 474 The TODO or SKIP variable, tors, should be lower case 475 """ 476 TORS=tors.upper() 477 template=eval("example_template."+tors+"line") 478 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 479 tab = '' 480 if reasons: 481 fh.write('if ! $force; then\n') 482 tab = tab + ' ' 483 if reasons == ["Requires DATAFILESPATH"]: 484 # The only reason not to run is DATAFILESPATH, which we check at run-time 485 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 486 tab = tab + ' ' 487 if reasons: 488 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 489 fh.write(tab+footer+"\n") 490 fh.write(tab+"exit\n") 491 if reasons == ["Requires DATAFILESPATH"]: 492 fh.write(' fi\n') 493 if reasons: 494 fh.write('fi\n') 495 fh.write('\n\n') 496 return 497 498 def getLoopVarsHead(self,loopVars,i,usedVars={}): 499 """ 500 Generate a nicely indented string with the format loops 501 Here is what the data structure looks like 502 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 503 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 504 loopVars['subargs']['pc_type']=["j","cholesky sor"] 505 """ 506 outstr=''; indnt=self.indent 507 508 for key in loopVars: 509 if key in usedVars: continue # Do not duplicate setting vars 510 for var in loopVars[key]['varlist']: 511 varval=loopVars[key][var] 512 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 513 outstr += "\n\n" 514 515 for key in loopVars: 516 for var in loopVars[key]['varlist']: 517 varval=loopVars[key][var] 518 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 519 i = i + 1 520 return (outstr,i) 521 522 def getLoopVarsFoot(self,loopVars,i): 523 outstr=''; indnt=self.indent 524 for key in loopVars: 525 for var in loopVars[key]['varlist']: 526 i = i - 1 527 outstr += indnt * i + "done\n" 528 return (outstr,i) 529 530 def genRunScript(self,testname,root,isRun,srcDict): 531 """ 532 Generate bash script using template found next to this file. 533 This file is read in at constructor time to avoid file I/O 534 """ 535 # runscript_dir directory has to be consistent with gmakefile 536 testDict=srcDict[testname] 537 rpath=self.srcrelpath(root) 538 runscript_dir=os.path.join(self.testroot_dir,rpath) 539 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 540 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 541 542 # Get variables to go into shell scripts. last time testDict used 543 subst=self.getSubstVars(testDict,rpath,testname) 544 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 545 if 'subtests' in testDict: 546 # The subtests inherit inDict, so we don't need top-level loops. 547 loopVars = {} 548 549 #Handle runfiles 550 for lfile in subst.get('localrunfiles','').split(): 551 install_files(os.path.join(root, lfile), 552 os.path.join(runscript_dir, os.path.dirname(lfile))) 553 # Check subtests for local runfiles 554 for stest in subst.get("subtests",[]): 555 for lfile in testDict[stest].get('localrunfiles','').split(): 556 install_files(os.path.join(root, lfile), 557 os.path.join(runscript_dir, os.path.dirname(lfile))) 558 559 # Now substitute the key variables into the header and footer 560 header=self._substVars(subst,example_template.header) 561 # The header is done twice to enable @...@ in header 562 header=self._substVars(subst,header) 563 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 564 565 # Start writing the file 566 fh.write(header+"\n") 567 568 # If there is a TODO or a SKIP then we do it before writing out the 569 # rest of the command (which is useful for working on the test) 570 # SKIP and TODO can be for the source file or for the runs 571 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 572 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 573 574 j=0 # for indentation 575 576 if loopVars: 577 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 578 if (loopHead): fh.write(loopHead+"\n") 579 580 # Subtests are special 581 allLoopVars=list(loopVars.keys()) 582 if 'subtests' in testDict: 583 substP=subst # Subtests can inherit args but be careful 584 k=0 # for label suffixes 585 for stest in testDict["subtests"]: 586 subst=substP.copy() 587 subst.update(testDict[stest]) 588 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 589 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 590 if sLoopVars: 591 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 592 allLoopVars+=list(sLoopVars.keys()) 593 fh.write(sLoopHead+"\n") 594 fh.write(self.getCmds(subst,j)+"\n") 595 if sLoopVars: 596 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 597 fh.write(sLoopFoot+"\n") 598 else: 599 fh.write(self.getCmds(subst,j)+"\n") 600 601 if loopVars: 602 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 603 fh.write(loopFoot+"\n") 604 605 fh.write(footer+"\n") 606 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 607 #if '10_9' in testname: sys.exit() 608 return 609 610 def genScriptsAndInfo(self,exfile,root,srcDict): 611 """ 612 Generate scripts from the source file, determine if built, etc. 613 For every test in the exfile with info in the srcDict: 614 1. Determine if it needs to be run for this arch 615 2. Generate the script 616 3. Generate the data needed to write out the makefile in a 617 convenient way 618 All tests are *always* run, but some may be SKIP'd per the TAP standard 619 """ 620 debug=False 621 rpath=self.srcrelpath(root) 622 execname=self.getExecname(exfile,rpath) 623 isBuilt=self._isBuilt(exfile,srcDict) 624 for test in srcDict: 625 if test in self.buildkeys: continue 626 if debug: print(self.nameSpace(exfile,root), test) 627 srcDict[test]['execname']=execname # Convenience in generating scripts 628 isRun=self._isRun(srcDict[test]) 629 self.genRunScript(test,root,isRun,srcDict) 630 srcDict[test]['isrun']=isRun 631 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 632 633 # This adds to datastructure for building deps 634 if isBuilt: self.addToSources(exfile,rpath,srcDict) 635 return 636 637 def _isBuilt(self,exfile,srcDict): 638 """ 639 Determine if this file should be built. 640 """ 641 # Get the language based on file extension 642 srcDict['SKIP'] = [] 643 lang=self.getLanguage(exfile) 644 if (lang=="F" or lang=="F90"): 645 if not self.have_fortran: 646 srcDict["SKIP"].append("Fortran required for this test") 647 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 648 srcDict["SKIP"].append("Fortran f90freeform required for this test") 649 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 650 srcDict["SKIP"].append("CUDA required for this test") 651 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 652 srcDict["SKIP"].append("C++ required for this test") 653 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 654 srcDict["SKIP"].append("C++ required for this test") 655 656 # Deprecated source files 657 if srcDict.get("TODO"): 658 return False 659 660 # isRun can work with srcDict to handle the requires 661 if "requires" in srcDict: 662 if srcDict["requires"]: 663 return self._isRun(srcDict) 664 665 return srcDict['SKIP'] == [] 666 667 668 def _isRun(self,testDict, debug=False): 669 """ 670 Based on the requirements listed in the src file and the petscconf.h 671 info, determine whether this test should be run or not. 672 """ 673 indent=" " 674 675 if 'SKIP' not in testDict: 676 testDict['SKIP'] = [] 677 # MPI requirements 678 if 'MPI_IS_MPIUNI' in self.conf: 679 if testDict.get('nsize', '1') != '1': 680 testDict['SKIP'].append("Parallel test with serial build") 681 682 # The requirements for the test are the sum of all the run subtests 683 if 'subtests' in testDict: 684 if 'requires' not in testDict: testDict['requires']="" 685 for stest in testDict['subtests']: 686 if 'requires' in testDict[stest]: 687 testDict['requires']+=" "+testDict[stest]['requires'] 688 if testDict.get('nsize', '1') != '1': 689 testDict['SKIP'].append("Parallel test with serial build") 690 break 691 692 # Now go through all requirements 693 if 'requires' in testDict: 694 for requirement in testDict['requires'].split(): 695 requirement=requirement.strip() 696 if not requirement: continue 697 if debug: print(indent+"Requirement: ", requirement) 698 isNull=False 699 if requirement.startswith("!"): 700 requirement=requirement[1:]; isNull=True 701 # Precision requirement for reals 702 if requirement in self.precision_types: 703 if self.conf['PETSC_PRECISION']==requirement: 704 if isNull: 705 testDict['SKIP'].append("not "+requirement+" required") 706 continue 707 continue # Success 708 elif not isNull: 709 testDict['SKIP'].append(requirement+" required") 710 continue 711 # Precision requirement for ints 712 if requirement in self.integer_types: 713 if requirement=="int32": 714 if self.conf['PETSC_SIZEOF_INT']==4: 715 if isNull: 716 testDict['SKIP'].append("not int32 required") 717 continue 718 continue # Success 719 elif not isNull: 720 testDict['SKIP'].append("int32 required") 721 continue 722 if requirement=="int64": 723 if self.conf['PETSC_SIZEOF_INT']==8: 724 if isNull: 725 testDict['SKIP'].append("NOT int64 required") 726 continue 727 continue # Success 728 elif not isNull: 729 testDict['SKIP'].append("int64 required") 730 continue 731 if requirement.startswith("long"): 732 reqsize = int(requirement[4:])//8 733 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 734 if longsize==reqsize: 735 if isNull: 736 testDict['SKIP'].append("not %s required" % requirement) 737 continue 738 continue # Success 739 elif not isNull: 740 testDict['SKIP'].append("%s required" % requirement) 741 continue 742 # Datafilespath 743 if requirement=="datafilespath" and not isNull: 744 testDict['SKIP'].append("Requires DATAFILESPATH") 745 continue 746 # Defines -- not sure I have comments matching 747 if "define(" in requirement.lower(): 748 reqdef=requirement.split("(")[1].split(")")[0] 749 if reqdef in self.conf: 750 if isNull: 751 testDict['SKIP'].append("Null requirement not met: "+requirement) 752 continue 753 continue # Success 754 elif not isNull: 755 testDict['SKIP'].append("Required: "+requirement) 756 continue 757 758 # Rest should be packages that we can just get from conf 759 if requirement == "complex": 760 petscconfvar="PETSC_USE_COMPLEX" 761 pkgconfvar="PETSC_USE_COMPLEX" 762 else: 763 petscconfvar="PETSC_HAVE_"+requirement.upper() 764 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 765 petsccv = self.conf.get(petscconfvar) 766 pkgcv = self.conf.get(pkgconfvar) 767 768 if petsccv or pkgcv: 769 if isNull: 770 if petsccv: 771 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 772 continue 773 else: 774 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 775 continue 776 continue # Success 777 elif not isNull: 778 if not petsccv and not pkgcv: 779 if debug: print("requirement not found: ", requirement) 780 if self.pkg_name == 'petsc': 781 testDict['SKIP'].append(petscconfvar+" requirement not met") 782 else: 783 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 784 continue 785 return testDict['SKIP'] == [] 786 787 def genPetscTests_summarize(self,dataDict): 788 """ 789 Required method to state what happened 790 """ 791 if not self.summarize: return 792 indent=" " 793 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 794 fh=open(fhname,"w") 795 for root in dataDict: 796 relroot=self.srcrelpath(root) 797 pkg=relroot.split("/")[1] 798 fh.write(relroot+"\n") 799 allSrcs=[] 800 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 801 for exfile in dataDict[root]: 802 # Basic information 803 rfile=os.path.join(relroot,exfile) 804 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 805 fh.write(indent+exfile+indent*4+builtStatus+"\n") 806 807 for test in dataDict[root][exfile]: 808 if test in self.buildkeys: continue 809 line=indent*2+test 810 fh.write(line+"\n") 811 # Looks nice to have the keys in order 812 #for key in dataDict[root][exfile][test]: 813 for key in "isrun abstracted nsize args requires script".split(): 814 if key not in dataDict[root][exfile][test]: continue 815 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 816 fh.write(line+"\n") 817 fh.write("\n") 818 fh.write("\n") 819 fh.write("\n") 820 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 821 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 822 fh.close() 823 return 824 825 def genPetscTests(self,root,dirs,files,dataDict): 826 """ 827 Go through and parse the source files in the directory to generate 828 the examples based on the metadata contained in the source files 829 """ 830 debug=False 831 # Use examplesAnalyze to get what the makefles think are sources 832 #self.examplesAnalyze(root,dirs,files,anlzDict) 833 834 dataDict[root]={} 835 836 for exfile in files: 837 #TST: Until we replace files, still leaving the orginals as is 838 #if not exfile.startswith("new_"+"ex"): continue 839 #if not exfile.startswith("ex"): continue 840 841 # Ignore emacs and other temporary files 842 if exfile.startswith("."): continue 843 if exfile.startswith("#"): continue 844 if exfile.endswith("~"): continue 845 # Only parse source files 846 ext=os.path.splitext(exfile)[-1].lstrip('.') 847 if ext not in LANGS: continue 848 849 # Convenience 850 fullex=os.path.join(root,exfile) 851 if self.verbose: print(' --> '+fullex) 852 dataDict[root].update(testparse.parseTestFile(fullex,0)) 853 if exfile in dataDict[root]: 854 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 855 856 return 857 858 def walktree(self,top): 859 """ 860 Walk a directory tree, starting from 'top' 861 """ 862 # Goal of action is to fill this dictionary 863 dataDict={} 864 for root, dirs, files in os.walk(top, topdown=True): 865 dirs.sort() 866 files.sort() 867 if not "examples" in root: continue 868 if "dSYM" in root: continue 869 if os.path.basename(root.rstrip("/")) == 'output': continue 870 if self.verbose: print(root) 871 self.genPetscTests(root,dirs,files,dataDict) 872 # Now summarize this dictionary 873 if self.verbose: self.genPetscTests_summarize(dataDict) 874 return dataDict 875 876 def gen_gnumake(self, fd): 877 """ 878 Overwrite of the method in the base PETSc class 879 """ 880 def write(stem, srcs): 881 for lang in LANGS: 882 if srcs[lang]['srcs']: 883 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 884 for pkg in self.pkg_pkgs: 885 srcs = self.gen_pkg(pkg) 886 write('testsrcs-' + pkg, srcs) 887 # Handle dependencies 888 for lang in LANGS: 889 for exfile in srcs[lang]['srcs']: 890 if exfile in srcs[lang]: 891 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 892 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 893 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 894 if deps: 895 # The executable literally depends on the object file because it is linked 896 fd.write(ex +": " + " ".join(deps) +'\n') 897 # The object file containing 'main' does not normally depend on other object 898 # files, but it does when it includes their modules. This dependency is 899 # overly blunt and could be reduced to only depend on object files for 900 # modules that are used, like "*f90aux.o". 901 fd.write(exfo +": " + " ".join(deps) +'\n') 902 903 return self.gendeps 904 905 def gen_pkg(self, pkg): 906 """ 907 Overwrite of the method in the base PETSc class 908 """ 909 return self.sources[pkg] 910 911 def write_gnumake(self, dataDict, output=None): 912 """ 913 Write out something similar to files from gmakegen.py 914 915 Test depends on script which also depends on source 916 file, but since I don't have a good way generating 917 acting on a single file (oops) just depend on 918 executable which in turn will depend on src file 919 """ 920 # Different options for how to set up the targets 921 compileExecsFirst=False 922 923 # Open file 924 fd = open(output, 'w') 925 926 # Write out the sources 927 gendeps = self.gen_gnumake(fd) 928 929 # Write out the tests and execname targets 930 fd.write("\n#Tests and executables\n") # Delimiter 931 932 for pkg in self.pkg_pkgs: 933 # These grab the ones that are built 934 for lang in LANGS: 935 testdeps=[] 936 for ftest in self.tests[pkg][lang]: 937 test=os.path.basename(ftest) 938 basedir=os.path.dirname(ftest) 939 testdeps.append(self.nameSpace(test,basedir)) 940 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 941 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 942 943 # test targets 944 for ftest in self.tests[pkg][lang]: 945 test=os.path.basename(ftest) 946 basedir=os.path.dirname(ftest) 947 testdir="${TESTDIR}/"+basedir+"/" 948 nmtest=self.nameSpace(test,basedir) 949 rundir=os.path.join(testdir,test) 950 script=test+".sh" 951 952 # Deps 953 exfile=self.tests[pkg][lang][ftest]['exfile'] 954 fullex=os.path.join(self.srcdir,exfile) 955 localexec=self.tests[pkg][lang][ftest]['exec'] 956 execname=os.path.join(testdir,localexec) 957 fullscript=os.path.join(testdir,script) 958 tmpfile=os.path.join(testdir,test,test+".tmp") 959 960 # *.counts depends on the script and either executable (will 961 # be run) or the example source file (SKIP or TODO) 962 fd.write('%s.counts : %s %s' 963 % (os.path.join('$(TESTDIR)/counts', nmtest), 964 fullscript, 965 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 966 ) 967 if exfile in self.sources[pkg][lang]: 968 for dep in self.sources[pkg][lang][exfile]: 969 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 970 fd.write('\n') 971 972 # Now write the args: 973 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 974 975 fd.close() 976 return 977 978def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 979 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 980 testdir=os.path.normpath(testdir) 981 if petsc_arch: 982 petsc_arch=petsc_arch.rstrip(os.path.sep) 983 if len(petsc_arch.split(os.path.sep))>1: 984 petsc_dir,petsc_arch=os.path.split(petsc_arch) 985 output = os.path.join(testdir, 'testfiles') 986 987 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 988 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 989 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 990 testdir=testdir) 991 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 992 pEx.write_gnumake(dataDict, output) 993 994if __name__ == '__main__': 995 import optparse 996 parser = optparse.OptionParser() 997 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 998 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 999 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1000 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1001 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1002 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1003 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1004 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1005 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1006 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1007 1008 opts, extra_args = parser.parse_args() 1009 if extra_args: 1010 import sys 1011 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1012 exit(1) 1013 if opts.testdir is None: 1014 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1015 1016 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1017 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1018 verbose=opts.verbose, 1019 single_ex=opts.single_executable, srcdir=opts.srcdir, 1020 testdir=opts.testdir) 1021