1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5import sys 6import logging, time 7import types 8sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 9from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77class generateExamples(Petsc): 78 """ 79 gmakegen.py has basic structure for finding the files, writing out 80 the dependencies, etc. 81 """ 82 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 83 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 84 85 self.single_ex=single_ex 86 self.srcdir=srcdir 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 if self.srcdir is None: 98 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 99 else: 100 if petsc_arch == '': 101 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 102 # Case 1 discussed above 103 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 104 if self.srcdir is None: 105 self.srcdir=os.path.join(self.petsc_dir,'src') 106 107 self.testroot_dir=os.path.abspath(testdir) 108 109 self.ptNaming=True 110 self.verbose=verbose 111 # Whether to write out a useful debugging 112 self.summarize=True if verbose else False 113 114 # For help in setting the requirements 115 self.precision_types="single double __float128 int32".split() 116 self.integer_types="int32 int64".split() 117 self.languages="fortran cuda cxx cpp".split() # Always requires C so do not list 118 119 # Things that are not test 120 self.buildkeys=testparse.buildkeys 121 122 # Adding a dictionary for storing sources, objects, and tests 123 # to make building the dependency tree easier 124 self.sources={} 125 self.objects={} 126 self.tests={} 127 for pkg in self.pkg_pkgs: 128 self.sources[pkg]={} 129 self.objects[pkg]=[] 130 self.tests[pkg]={} 131 for lang in LANGS: 132 self.sources[pkg][lang]={} 133 self.sources[pkg][lang]['srcs']=[] 134 self.tests[pkg][lang]={} 135 136 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 137 138 self.indent=" " 139 if self.verbose: print('Finishing the constructor') 140 return 141 142 def srcrelpath(self,rdir): 143 """ 144 Get relative path to source directory 145 """ 146 return os.path.relpath(rdir,self.srcdir) 147 148 def getInInstallDir(self,thisscriptdir): 149 """ 150 When petsc is installed then this file in installed in: 151 <PREFIX>/share/petsc/examples/config/gmakegentest.py 152 otherwise the path is: 153 <PETSC_DIR>/config/gmakegentest.py 154 We use this difference to determine if we are in installdir 155 """ 156 dirlist=thisscriptdir.split(os.path.sep) 157 if len(dirlist)>4: 158 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 159 if lastfour==os.path.join('share','petsc','examples','config'): 160 return True 161 else: 162 return False 163 else: 164 return False 165 166 def nameSpace(self,srcfile,srcdir): 167 """ 168 Because the scripts have a non-unique naming, the pretty-printing 169 needs to convey the srcdir and srcfile. There are two ways of doing this. 170 """ 171 if self.ptNaming: 172 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 173 cdir=srcdir 174 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 175 nameString=prefix+srcfile 176 else: 177 #nameString=srcdir+": "+srcfile 178 nameString=srcfile 179 return nameString 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:\] now? 185 """ 186 langReq=None 187 srcext=os.path.splitext(srcfile)[-1] 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getLoopVars(self,inDict,testname, isSubtest=False): 198 """ 199 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 200 Return: 201 inDict['args']: -ksp_monitor 202 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 203 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 204 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 205 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 206 subst should be passed in instead of inDict 207 """ 208 loopVars={}; newargs=[] 209 lsuffix='_' 210 argregex = re.compile(' (?=-[a-zA-Z])') 211 from testparse import parseLoopArgs 212 for key in inDict: 213 if key in ('SKIP', 'regexes'): 214 continue 215 akey=('subargs' if key=='args' else key) # what to assign 216 if akey not in inDict: inDict[akey]='' 217 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 218 # Always generate a loop over nsize, even if there is only one value 219 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 220 keystr = str(inDict[key]) 221 varlist = [] 222 for varset in argregex.split(keystr): 223 if not varset.strip(): continue 224 if '{{' in varset: 225 keyvar,lvars,ftype=parseLoopArgs(varset) 226 if akey not in loopVars: loopVars[akey]={} 227 varlist.append(keyvar) 228 loopVars[akey][keyvar]=[keyvar,lvars] 229 if akey=='nsize': 230 if len(lvars.split()) > 1: 231 lsuffix += akey +'-${' + keyvar + '}' 232 else: 233 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 234 lsuffix+=keyvar+'-${' + keyvar + '}_' 235 else: 236 if key=='args': 237 newargs.append(varset.strip()) 238 if varlist: 239 loopVars[akey]['varlist']=varlist 240 241 # For subtests, args are always substituted in (not top level) 242 if isSubtest: 243 inDict['subargs'] += " ".join(newargs) 244 inDict['args']='' 245 if 'label_suffix' in inDict: 246 inDict['label_suffix']+=lsuffix.rstrip('_') 247 else: 248 inDict['label_suffix']=lsuffix.rstrip('_') 249 else: 250 if loopVars: 251 inDict['args'] = ' '.join(newargs) 252 inDict['label_suffix']=lsuffix.rstrip('_') 253 return loopVars 254 255 def getArgLabel(self,testDict): 256 """ 257 In all of the arguments in the test dictionary, create a simple 258 string for searching within the makefile system. For simplicity in 259 search, remove "-", for strings, etc. 260 Also, concatenate the arg commands 261 For now, ignore nsize -- seems hard to search for anyway 262 """ 263 # Collect all of the args associated with a test 264 argStr=("" if 'args' not in testDict else testDict['args']) 265 if 'subtests' in testDict: 266 for stest in testDict["subtests"]: 267 sd=testDict[stest] 268 argStr=argStr+("" if 'args' not in sd else sd['args']) 269 270 # Now go through and cleanup 271 argStr=re.sub('{{(.*?)}}',"",argStr) 272 argStr=re.sub('-'," ",argStr) 273 for digit in string.digits: argStr=re.sub(digit," ",argStr) 274 argStr=re.sub("\.","",argStr) 275 argStr=re.sub(",","",argStr) 276 argStr=re.sub('\+',' ',argStr) 277 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 278 return argStr.strip() 279 280 def addToSources(self,exfile,rpath,srcDict): 281 """ 282 Put into data structure that allows easy generation of makefile 283 """ 284 pkg=rpath.split(os.path.sep)[0] 285 relpfile=os.path.join(rpath,exfile) 286 lang=self.getLanguage(exfile) 287 if not lang: return 288 if pkg not in self.sources: return 289 self.sources[pkg][lang]['srcs'].append(relpfile) 290 self.sources[pkg][lang][relpfile] = [] 291 if 'depends' in srcDict: 292 depSrcList=srcDict['depends'].split() 293 for depSrc in depSrcList: 294 depObj=os.path.splitext(depSrc)[0]+".o" 295 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 296 297 # In gmakefile, ${TESTDIR} var specifies the object compilation 298 testsdir=rpath+"/" 299 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 300 self.objects[pkg].append(objfile) 301 return 302 303 def addToTests(self,test,rpath,exfile,execname,testDict): 304 """ 305 Put into data structure that allows easy generation of makefile 306 Organized by languages to allow testing of languages 307 """ 308 pkg=rpath.split("/")[0] 309 nmtest=os.path.join(rpath,test) 310 lang=self.getLanguage(exfile) 311 if not lang: return 312 if pkg not in self.tests: return 313 self.tests[pkg][lang][nmtest]={} 314 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 315 self.tests[pkg][lang][nmtest]['exec']=execname 316 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 317 return 318 319 def getExecname(self,exfile,rpath): 320 """ 321 Generate bash script using template found next to this file. 322 This file is read in at constructor time to avoid file I/O 323 """ 324 if self.single_ex: 325 execname=rpath.split("/")[1]+"-ex" 326 else: 327 execname=os.path.splitext(exfile)[0] 328 return execname 329 330 def getSubstVars(self,testDict,rpath,testname): 331 """ 332 Create a dictionary with all of the variables that get substituted 333 into the template commands found in example_template.py 334 """ 335 subst={} 336 337 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 338 if 'nsize' not in testDict: testDict['nsize'] = '1' 339 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 340 for ak in testparse.acceptedkeys: 341 if ak=='test': continue 342 subst[ak]=(testDict[ak] if ak in testDict else '') 343 344 # Now do other variables 345 subst['execname']=testDict['execname'] 346 if 'filter' in testDict: 347 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 348 349 # Others 350 subst['subargs']='' # Default. For variables override 351 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 352 subst['label_suffix']='' 353 subst['comments']="\n#".join(subst['comments'].split("\n")) 354 if subst['comments']: subst['comments']="#"+subst['comments'] 355 subst['exec']="../"+subst['execname'] 356 subst['testroot']=self.testroot_dir 357 subst['testname']=testname 358 dp = self.conf.get('DATAFILESPATH','') 359 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 360 361 # This is used to label some matrices 362 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 363 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 364 365 #Conf vars 366 if self.petsc_arch.find('valgrind')>=0: 367 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 368 else: 369 subst['mpiexec']=self.conf['MPIEXEC'] 370 subst['pkg_name']=self.pkg_name 371 subst['pkg_dir']=self.pkg_dir 372 subst['pkg_arch']=self.petsc_arch 373 374 if self.inInstallDir: 375 # Case 2 376 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 377 else: 378 # Case 1 379 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 380 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 381 subst['diff']=self.conf['DIFF'] 382 subst['rm']=self.conf['RM'] 383 subst['grep']=self.conf['GREP'] 384 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 385 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 386 387 # Output file is special because of subtests override 388 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 389 if not "_" in defroot: defroot=defroot+"_1" 390 subst['defroot']=defroot 391 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 392 subst['redirect_file']=defroot+".tmp" 393 if 'output_file' not in testDict: 394 subst['output_file']="output/"+defroot+".out" 395 # Add in the full path here. 396 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 397 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 398 if not subst['TODO']: 399 print("Warning: "+subst['output_file']+" not found.") 400 # Worry about alt files here -- see 401 # src/snes/examples/tutorials/output/ex22*.out 402 altlist=[subst['output_file']] 403 basefile,ext = os.path.splitext(subst['output_file']) 404 for i in range(1,9): 405 altroot=basefile+"_alt" 406 if i > 1: altroot=altroot+"_"+str(i) 407 af=altroot+".out" 408 srcaf=os.path.join(subst['srcdir'],af) 409 fullaf=os.path.join(self.petsc_dir,srcaf) 410 if os.path.isfile(fullaf): altlist.append(srcaf) 411 if len(altlist)>1: subst['altfiles']=altlist 412 #if len(altlist)>1: print("Found alt files: ",altlist) 413 414 subst['regexes']={} 415 for subkey in subst: 416 if subkey=='regexes': continue 417 if not isinstance(subst[subkey],str): continue 418 patt="@"+subkey.upper()+"@" 419 subst['regexes'][subkey]=re.compile(patt) 420 421 return subst 422 423 def _substVars(self,subst,origStr): 424 """ 425 Substitute variables 426 """ 427 Str=origStr 428 for subkey in subst: 429 if subkey=='regexes': continue 430 if not isinstance(subst[subkey],str): continue 431 if subkey.upper() not in Str: continue 432 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 433 return Str 434 435 def getCmds(self,subst,i): 436 """ 437 Generate bash script using template found next to this file. 438 This file is read in at constructor time to avoid file I/O 439 """ 440 nindnt=i # the start and has to be consistent with below 441 cmdindnt=self.indent*nindnt 442 cmdLines="" 443 444 # MPI is the default -- but we have a few odd commands 445 if not subst['command']: 446 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 447 else: 448 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 449 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 450 451 cmdLines+=cmdindnt+'if test $res = 0; then\n' 452 diffindnt=self.indent*(nindnt+1) 453 if not subst['filter_output']: 454 if 'altfiles' not in subst: 455 cmd=diffindnt+self._substVars(subst,example_template.difftest) 456 else: 457 # Have to do it by hand a bit because of variable number of alt files 458 rf=subst['redirect_file'] 459 cmd=diffindnt+example_template.difftest.split('@')[0] 460 for i in range(len(subst['altfiles'])): 461 af=subst['altfiles'][i] 462 cmd+=af+' '+rf 463 if i!=len(subst['altfiles'])-1: 464 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 465 cmd+=' || ${diff_exe} ' 466 else: 467 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 468 cmd+=subst['label_suffix']+' ""' # Quotes are painful 469 else: 470 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 471 cmdLines+=cmd+"\n" 472 cmdLines+=cmdindnt+'else\n' 473 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 474 cmdLines+=cmdindnt+'fi\n' 475 return cmdLines 476 477 def _writeTodoSkip(self,fh,tors,reasons,footer): 478 """ 479 Write out the TODO and SKIP lines in the file 480 The TODO or SKIP variable, tors, should be lower case 481 """ 482 TORS=tors.upper() 483 template=eval("example_template."+tors+"line") 484 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 485 tab = '' 486 if reasons: 487 fh.write('if ! $force; then\n') 488 tab = tab + ' ' 489 if reasons == ["Requires DATAFILESPATH"]: 490 # The only reason not to run is DATAFILESPATH, which we check at run-time 491 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 492 tab = tab + ' ' 493 if reasons: 494 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 495 fh.write(tab+footer+"\n") 496 fh.write(tab+"exit\n") 497 if reasons == ["Requires DATAFILESPATH"]: 498 fh.write(' fi\n') 499 if reasons: 500 fh.write('fi\n') 501 fh.write('\n\n') 502 return 503 504 def getLoopVarsHead(self,loopVars,i,usedVars={}): 505 """ 506 Generate a nicely indented string with the format loops 507 Here is what the data structure looks like 508 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 509 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 510 loopVars['subargs']['pc_type']=["j","cholesky sor"] 511 """ 512 outstr=''; indnt=self.indent 513 514 for key in loopVars: 515 if key in usedVars: continue # Do not duplicate setting vars 516 for var in loopVars[key]['varlist']: 517 varval=loopVars[key][var] 518 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 519 outstr += "\n\n" 520 521 for key in loopVars: 522 for var in loopVars[key]['varlist']: 523 varval=loopVars[key][var] 524 outstr += indnt * i + "for {0} in ${{{0}_in}}; do\n".format(*varval) 525 i = i + 1 526 return (outstr,i) 527 528 def getLoopVarsFoot(self,loopVars,i): 529 outstr=''; indnt=self.indent 530 for key in loopVars: 531 for var in loopVars[key]['varlist']: 532 i = i - 1 533 outstr += indnt * i + "done\n" 534 return (outstr,i) 535 536 def genRunScript(self,testname,root,isRun,srcDict): 537 """ 538 Generate bash script using template found next to this file. 539 This file is read in at constructor time to avoid file I/O 540 """ 541 # runscript_dir directory has to be consistent with gmakefile 542 testDict=srcDict[testname] 543 rpath=self.srcrelpath(root) 544 runscript_dir=os.path.join(self.testroot_dir,rpath) 545 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 546 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 547 548 # Get variables to go into shell scripts. last time testDict used 549 subst=self.getSubstVars(testDict,rpath,testname) 550 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 551 if 'subtests' in testDict: 552 # The subtests inherit inDict, so we don't need top-level loops. 553 loopVars = {} 554 555 #Handle runfiles 556 for lfile in subst.get('localrunfiles','').split(): 557 install_files(os.path.join(root, lfile), 558 os.path.join(runscript_dir, os.path.dirname(lfile))) 559 # Check subtests for local runfiles 560 for stest in subst.get("subtests",[]): 561 for lfile in testDict[stest].get('localrunfiles','').split(): 562 install_files(os.path.join(root, lfile), 563 os.path.join(runscript_dir, os.path.dirname(lfile))) 564 565 # Now substitute the key variables into the header and footer 566 header=self._substVars(subst,example_template.header) 567 # The header is done twice to enable @...@ in header 568 header=self._substVars(subst,header) 569 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 570 571 # Start writing the file 572 fh.write(header+"\n") 573 574 # If there is a TODO or a SKIP then we do it before writing out the 575 # rest of the command (which is useful for working on the test) 576 # SKIP and TODO can be for the source file or for the runs 577 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 578 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 579 580 j=0 # for indentation 581 582 if loopVars: 583 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 584 if (loopHead): fh.write(loopHead+"\n") 585 586 # Subtests are special 587 allLoopVars=list(loopVars.keys()) 588 if 'subtests' in testDict: 589 substP=subst # Subtests can inherit args but be careful 590 k=0 # for label suffixes 591 for stest in testDict["subtests"]: 592 subst=substP.copy() 593 subst.update(testDict[stest]) 594 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 595 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 596 if sLoopVars: 597 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 598 allLoopVars+=list(sLoopVars.keys()) 599 fh.write(sLoopHead+"\n") 600 fh.write(self.getCmds(subst,j)+"\n") 601 if sLoopVars: 602 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 603 fh.write(sLoopFoot+"\n") 604 else: 605 fh.write(self.getCmds(subst,j)+"\n") 606 607 if loopVars: 608 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 609 fh.write(loopFoot+"\n") 610 611 fh.write(footer+"\n") 612 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 613 #if '10_9' in testname: sys.exit() 614 return 615 616 def genScriptsAndInfo(self,exfile,root,srcDict): 617 """ 618 Generate scripts from the source file, determine if built, etc. 619 For every test in the exfile with info in the srcDict: 620 1. Determine if it needs to be run for this arch 621 2. Generate the script 622 3. Generate the data needed to write out the makefile in a 623 convenient way 624 All tests are *always* run, but some may be SKIP'd per the TAP standard 625 """ 626 debug=False 627 rpath=self.srcrelpath(root) 628 execname=self.getExecname(exfile,rpath) 629 isBuilt=self._isBuilt(exfile,srcDict) 630 for test in srcDict: 631 if test in self.buildkeys: continue 632 if debug: print(self.nameSpace(exfile,root), test) 633 srcDict[test]['execname']=execname # Convenience in generating scripts 634 isRun=self._isRun(srcDict[test]) 635 self.genRunScript(test,root,isRun,srcDict) 636 srcDict[test]['isrun']=isRun 637 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 638 639 # This adds to datastructure for building deps 640 if isBuilt: self.addToSources(exfile,rpath,srcDict) 641 return 642 643 def _isBuilt(self,exfile,srcDict): 644 """ 645 Determine if this file should be built. 646 """ 647 # Get the language based on file extension 648 srcDict['SKIP'] = [] 649 lang=self.getLanguage(exfile) 650 if (lang=="F" or lang=="F90"): 651 if not self.have_fortran: 652 srcDict["SKIP"].append("Fortran required for this test") 653 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 654 srcDict["SKIP"].append("Fortran f90freeform required for this test") 655 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 656 srcDict["SKIP"].append("CUDA required for this test") 657 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 658 srcDict["SKIP"].append("C++ required for this test") 659 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 660 srcDict["SKIP"].append("C++ required for this test") 661 662 # Deprecated source files 663 if srcDict.get("TODO"): 664 return False 665 666 # isRun can work with srcDict to handle the requires 667 if "requires" in srcDict: 668 if srcDict["requires"]: 669 return self._isRun(srcDict) 670 671 return srcDict['SKIP'] == [] 672 673 674 def _isRun(self,testDict, debug=False): 675 """ 676 Based on the requirements listed in the src file and the petscconf.h 677 info, determine whether this test should be run or not. 678 """ 679 indent=" " 680 681 if 'SKIP' not in testDict: 682 testDict['SKIP'] = [] 683 # MPI requirements 684 if 'MPI_IS_MPIUNI' in self.conf: 685 if testDict.get('nsize', '1') != '1': 686 testDict['SKIP'].append("Parallel test with serial build") 687 688 # The requirements for the test are the sum of all the run subtests 689 if 'subtests' in testDict: 690 if 'requires' not in testDict: testDict['requires']="" 691 for stest in testDict['subtests']: 692 if 'requires' in testDict[stest]: 693 testDict['requires']+=" "+testDict[stest]['requires'] 694 if testDict.get('nsize', '1') != '1': 695 testDict['SKIP'].append("Parallel test with serial build") 696 break 697 698 # Now go through all requirements 699 if 'requires' in testDict: 700 for requirement in testDict['requires'].split(): 701 requirement=requirement.strip() 702 if not requirement: continue 703 if debug: print(indent+"Requirement: ", requirement) 704 isNull=False 705 if requirement.startswith("!"): 706 requirement=requirement[1:]; isNull=True 707 # Precision requirement for reals 708 if requirement in self.precision_types: 709 if self.conf['PETSC_PRECISION']==requirement: 710 if isNull: 711 testDict['SKIP'].append("not "+requirement+" required") 712 continue 713 continue # Success 714 elif not isNull: 715 testDict['SKIP'].append(requirement+" required") 716 continue 717 # Precision requirement for ints 718 if requirement in self.integer_types: 719 if requirement=="int32": 720 if self.conf['PETSC_SIZEOF_INT']==4: 721 if isNull: 722 testDict['SKIP'].append("not int32 required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append("int32 required") 727 continue 728 if requirement=="int64": 729 if self.conf['PETSC_SIZEOF_INT']==8: 730 if isNull: 731 testDict['SKIP'].append("NOT int64 required") 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append("int64 required") 736 continue 737 # Datafilespath 738 if requirement=="datafilespath" and not isNull: 739 testDict['SKIP'].append("Requires DATAFILESPATH") 740 continue 741 # Defines -- not sure I have comments matching 742 if "define(" in requirement.lower(): 743 reqdef=requirement.split("(")[1].split(")")[0] 744 if reqdef in self.conf: 745 if isNull: 746 testDict['SKIP'].append("Null requirement not met: "+requirement) 747 continue 748 continue # Success 749 elif not isNull: 750 testDict['SKIP'].append("Required: "+requirement) 751 continue 752 753 # Rest should be packages that we can just get from conf 754 if requirement == "complex": 755 petscconfvar="PETSC_USE_COMPLEX" 756 pkgconfvar="PETSC_USE_COMPLEX" 757 else: 758 petscconfvar="PETSC_HAVE_"+requirement.upper() 759 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 760 petsccv = self.conf.get(petscconfvar) 761 pkgcv = self.conf.get(pkgconfvar) 762 763 if petsccv or pkgcv: 764 if isNull: 765 if petsccv: 766 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 767 continue 768 else: 769 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 770 continue 771 continue # Success 772 elif not isNull: 773 if not petsccv and not pkgcv: 774 if debug: print("requirement not found: ", requirement) 775 if self.pkg_name == 'petsc': 776 testDict['SKIP'].append(petscconfvar+" requirement not met") 777 else: 778 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 779 continue 780 return testDict['SKIP'] == [] 781 782 def genPetscTests_summarize(self,dataDict): 783 """ 784 Required method to state what happened 785 """ 786 if not self.summarize: return 787 indent=" " 788 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 789 fh=open(fhname,"w") 790 for root in dataDict: 791 relroot=self.srcrelpath(root) 792 pkg=relroot.split("/")[1] 793 fh.write(relroot+"\n") 794 allSrcs=[] 795 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 796 for exfile in dataDict[root]: 797 # Basic information 798 rfile=os.path.join(relroot,exfile) 799 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 800 fh.write(indent+exfile+indent*4+builtStatus+"\n") 801 802 for test in dataDict[root][exfile]: 803 if test in self.buildkeys: continue 804 line=indent*2+test 805 fh.write(line+"\n") 806 # Looks nice to have the keys in order 807 #for key in dataDict[root][exfile][test]: 808 for key in "isrun abstracted nsize args requires script".split(): 809 if key not in dataDict[root][exfile][test]: continue 810 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 811 fh.write(line+"\n") 812 fh.write("\n") 813 fh.write("\n") 814 fh.write("\n") 815 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 816 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 817 fh.close() 818 return 819 820 def genPetscTests(self,root,dirs,files,dataDict): 821 """ 822 Go through and parse the source files in the directory to generate 823 the examples based on the metadata contained in the source files 824 """ 825 debug=False 826 # Use examplesAnalyze to get what the makefles think are sources 827 #self.examplesAnalyze(root,dirs,files,anlzDict) 828 829 dataDict[root]={} 830 831 for exfile in files: 832 #TST: Until we replace files, still leaving the orginals as is 833 #if not exfile.startswith("new_"+"ex"): continue 834 #if not exfile.startswith("ex"): continue 835 836 # Ignore emacs and other temporary files 837 if exfile.startswith("."): continue 838 if exfile.startswith("#"): continue 839 if exfile.endswith("~"): continue 840 # Only parse source files 841 ext=os.path.splitext(exfile)[-1].lstrip('.') 842 if ext not in LANGS: continue 843 844 # Convenience 845 fullex=os.path.join(root,exfile) 846 if self.verbose: print(' --> '+fullex) 847 dataDict[root].update(testparse.parseTestFile(fullex,0)) 848 if exfile in dataDict[root]: 849 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 850 851 return 852 853 def walktree(self,top): 854 """ 855 Walk a directory tree, starting from 'top' 856 """ 857 # Goal of action is to fill this dictionary 858 dataDict={} 859 for root, dirs, files in os.walk(top, topdown=True): 860 dirs.sort() 861 files.sort() 862 if not "examples" in root: continue 863 if "dSYM" in root: continue 864 if os.path.basename(root.rstrip("/")) == 'output': continue 865 if self.verbose: print(root) 866 self.genPetscTests(root,dirs,files,dataDict) 867 # Now summarize this dictionary 868 if self.verbose: self.genPetscTests_summarize(dataDict) 869 return dataDict 870 871 def gen_gnumake(self, fd): 872 """ 873 Overwrite of the method in the base PETSc class 874 """ 875 def write(stem, srcs): 876 for lang in LANGS: 877 if srcs[lang]['srcs']: 878 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 879 for pkg in self.pkg_pkgs: 880 srcs = self.gen_pkg(pkg) 881 write('testsrcs-' + pkg, srcs) 882 # Handle dependencies 883 for lang in LANGS: 884 for exfile in srcs[lang]['srcs']: 885 if exfile in srcs[lang]: 886 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 887 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 888 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 889 if deps: 890 # The executable literally depends on the object file because it is linked 891 fd.write(ex +": " + " ".join(deps) +'\n') 892 # The object file containing 'main' does not normally depend on other object 893 # files, but it does when it includes their modules. This dependency is 894 # overly blunt and could be reduced to only depend on object files for 895 # modules that are used, like "*f90aux.o". 896 fd.write(exfo +": " + " ".join(deps) +'\n') 897 898 return self.gendeps 899 900 def gen_pkg(self, pkg): 901 """ 902 Overwrite of the method in the base PETSc class 903 """ 904 return self.sources[pkg] 905 906 def write_gnumake(self, dataDict, output=None): 907 """ 908 Write out something similar to files from gmakegen.py 909 910 Test depends on script which also depends on source 911 file, but since I don't have a good way generating 912 acting on a single file (oops) just depend on 913 executable which in turn will depend on src file 914 """ 915 # Different options for how to set up the targets 916 compileExecsFirst=False 917 918 # Open file 919 fd = open(output, 'w') 920 921 # Write out the sources 922 gendeps = self.gen_gnumake(fd) 923 924 # Write out the tests and execname targets 925 fd.write("\n#Tests and executables\n") # Delimiter 926 927 for pkg in self.pkg_pkgs: 928 # These grab the ones that are built 929 for lang in LANGS: 930 testdeps=[] 931 for ftest in self.tests[pkg][lang]: 932 test=os.path.basename(ftest) 933 basedir=os.path.dirname(ftest) 934 testdeps.append(self.nameSpace(test,basedir)) 935 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 936 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 937 938 # test targets 939 for ftest in self.tests[pkg][lang]: 940 test=os.path.basename(ftest) 941 basedir=os.path.dirname(ftest) 942 testdir="${TESTDIR}/"+basedir+"/" 943 nmtest=self.nameSpace(test,basedir) 944 rundir=os.path.join(testdir,test) 945 script=test+".sh" 946 947 # Deps 948 exfile=self.tests[pkg][lang][ftest]['exfile'] 949 fullex=os.path.join(self.srcdir,exfile) 950 localexec=self.tests[pkg][lang][ftest]['exec'] 951 execname=os.path.join(testdir,localexec) 952 fullscript=os.path.join(testdir,script) 953 tmpfile=os.path.join(testdir,test,test+".tmp") 954 955 # *.counts depends on the script and either executable (will 956 # be run) or the example source file (SKIP or TODO) 957 fd.write('%s.counts : %s %s' 958 % (os.path.join('$(TESTDIR)/counts', nmtest), 959 fullscript, 960 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 961 ) 962 if exfile in self.sources[pkg][lang]: 963 for dep in self.sources[pkg][lang][exfile]: 964 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 965 fd.write('\n') 966 967 # Now write the args: 968 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 969 970 fd.close() 971 return 972 973def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 974 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 975 testdir=os.path.normpath(testdir) 976 if petsc_arch: 977 petsc_arch=petsc_arch.rstrip(os.path.sep) 978 if len(petsc_arch.split(os.path.sep))>1: 979 petsc_dir,petsc_arch=os.path.split(petsc_arch) 980 output = os.path.join(testdir, 'testfiles') 981 982 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 983 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 984 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 985 testdir=testdir) 986 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 987 pEx.write_gnumake(dataDict, output) 988 989if __name__ == '__main__': 990 import optparse 991 parser = optparse.OptionParser() 992 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 993 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 994 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 995 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 996 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 997 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 998 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 999 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1000 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1001 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1002 1003 opts, extra_args = parser.parse_args() 1004 if extra_args: 1005 import sys 1006 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1007 exit(1) 1008 if opts.testdir is None: 1009 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1010 1011 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1012 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1013 verbose=opts.verbose, 1014 single_ex=opts.single_executable, srcdir=opts.srcdir, 1015 testdir=opts.testdir) 1016