1#!/usr/bin/env python 2 3from __future__ import print_function 4import os,shutil, string, re 5from distutils.sysconfig import parse_makefile 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from cmakegen import Mistakes, stripsplit, AUTODIRS, SKIPDIRS 11from cmakegen import defaultdict # collections.defaultdict, with fallback for python-2.4 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78class generateExamples(Petsc): 79 """ 80 gmakegen.py has basic structure for finding the files, writing out 81 the dependencies, etc. 82 """ 83 def __init__(self,petsc_dir=None, petsc_arch=None, testdir='tests', verbose=False, single_ex=False, srcdir=None): 84 super(generateExamples, self).__init__(petsc_dir, petsc_arch, verbose) 85 86 self.single_ex=single_ex 87 88 # Set locations to handle movement 89 self.inInstallDir=self.getInInstallDir(thisscriptdir) 90 91 if self.inInstallDir: 92 # Case 2 discussed above 93 # set PETSC_ARCH to install directory to allow script to work in both 94 dirlist=thisscriptdir.split(os.path.sep) 95 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 96 self.arch_dir=installdir 97 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 98 else: 99 if petsc_arch == '': 100 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 101 # Case 1 discussed above 102 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 103 self.srcdir=os.path.join(self.petsc_dir,'src') 104 105 self.testroot_dir=os.path.abspath(testdir) 106 107 self.ptNaming=True 108 self.verbose=verbose 109 # Whether to write out a useful debugging 110 self.summarize=True if verbose else False 111 112 # For help in setting the requirements 113 self.precision_types="single double __float128 int32".split() 114 self.integer_types="int32 int64".split() 115 self.languages="fortran cuda cxx".split() # Always requires C so do not list 116 117 # Things that are not test 118 self.buildkeys=testparse.buildkeys 119 120 # Adding a dictionary for storing sources, objects, and tests 121 # to make building the dependency tree easier 122 self.sources={} 123 self.objects={} 124 self.tests={} 125 for pkg in PKGS: 126 self.sources[pkg]={} 127 self.objects[pkg]=[] 128 self.tests[pkg]={} 129 for lang in LANGS: 130 self.sources[pkg][lang]={} 131 self.sources[pkg][lang]['srcs']=[] 132 self.tests[pkg][lang]={} 133 134 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 135 136 self.indent=" " 137 if self.verbose: print('Finishing the constructor') 138 return 139 140 def srcrelpath(self,rdir): 141 """ 142 Get relative path to source directory 143 """ 144 return os.path.relpath(rdir,self.srcdir) 145 146 def getInInstallDir(self,thisscriptdir): 147 """ 148 When petsc is installed then this file in installed in: 149 <PREFIX>/share/petsc/examples/config/gmakegentest.py 150 otherwise the path is: 151 <PETSC_DIR>/config/gmakegentest.py 152 We use this difference to determine if we are in installdir 153 """ 154 dirlist=thisscriptdir.split(os.path.sep) 155 if len(dirlist)>4: 156 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 157 if lastfour==os.path.join('share','petsc','examples','config'): 158 return True 159 else: 160 return False 161 else: 162 return False 163 164 def nameSpace(self,srcfile,srcdir): 165 """ 166 Because the scripts have a non-unique naming, the pretty-printing 167 needs to convey the srcdir and srcfile. There are two ways of doing this. 168 """ 169 if self.ptNaming: 170 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 171 cdir=srcdir 172 prefix=cdir.replace('/examples/','_').replace("/","_")+"-" 173 nameString=prefix+srcfile 174 else: 175 #nameString=srcdir+": "+srcfile 176 nameString=srcfile 177 return nameString 178 179 def getLanguage(self,srcfile): 180 """ 181 Based on the source, determine associated language as found in gmakegen.LANGS 182 Can we just return srcext[1:\] now? 183 """ 184 langReq=None 185 srcext=os.path.splitext(srcfile)[-1] 186 if srcext in ".F90".split(): langReq="F90" 187 if srcext in ".F".split(): langReq="F" 188 if srcext in ".cxx".split(): langReq="cxx" 189 if srcext == ".cu": langReq="cu" 190 if srcext == ".c": langReq="c" 191 #if not langReq: print("ERROR: ", srcext, srcfile) 192 return langReq 193 194 def _getLoopVars(self,inDict,testname, isSubtest=False): 195 """ 196 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 197 Return: 198 inDict['args']: -ksp_monitor 199 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 200 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 201 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 202 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 203 subst should be passed in instead of inDict 204 """ 205 loopVars={}; newargs="" 206 lkeys=inDict.keys() 207 lsuffix='_' 208 argregex=re.compile('(?<![a-zA-Z])-(?=[a-zA-Z])') 209 from testparse import parseLoopArgs 210 for key in lkeys: 211 if type(inDict[key])!=bytes: continue 212 keystr = str(inDict[key]) 213 akey=('subargs' if key=='args' else key) # what to assign 214 if akey not in inDict: inDict[akey]='' 215 varlist=[] 216 for varset in argregex.split(keystr): 217 if not varset.strip(): continue 218 if '{{' in varset: 219 keyvar,lvars,ftype=parseLoopArgs(varset) 220 if akey not in loopVars: loopVars[akey]={} 221 varlist.append(keyvar) 222 loopVars[akey][keyvar]=[keyvar,lvars] 223 if akey=='nsize': 224 inDict[akey] = '${' + keyvar + '}' 225 lsuffix+=akey+'-'+inDict[akey]+'_' 226 else: 227 inDict[akey] += ' -'+keyvar+' ${' + keyvar + '}' 228 lsuffix+=keyvar+'-${' + keyvar + '}_' 229 else: 230 if key=='args': newargs+=" -"+varset.strip() 231 if varlist: loopVars[akey]['varlist']=varlist 232 233 234 # For subtests, args are always substituted in (not top level) 235 if isSubtest: 236 inDict['subargs']+=" "+newargs.strip() 237 inDict['args']='' 238 if 'label_suffix' in inDict: 239 inDict['label_suffix']+=lsuffix.rstrip('_') 240 else: 241 inDict['label_suffix']=lsuffix.rstrip('_') 242 else: 243 if loopVars.keys(): 244 inDict['args']=newargs.strip() 245 inDict['label_suffix']=lsuffix.rstrip('_') 246 if loopVars.keys(): 247 return loopVars 248 else: 249 return None 250 251 def getArgLabel(self,testDict): 252 """ 253 In all of the arguments in the test dictionary, create a simple 254 string for searching within the makefile system. For simplicity in 255 search, remove "-", for strings, etc. 256 Also, concatenate the arg commands 257 For now, ignore nsize -- seems hard to search for anyway 258 """ 259 # Collect all of the args associated with a test 260 argStr=("" if 'args' not in testDict else testDict['args']) 261 if 'subtests' in testDict: 262 for stest in testDict["subtests"]: 263 sd=testDict[stest] 264 argStr=argStr+("" if 'args' not in sd else sd['args']) 265 266 # Now go through and cleanup 267 argStr=re.sub('{{(.*?)}}',"",argStr) 268 argStr=re.sub('-'," ",argStr) 269 for digit in string.digits: argStr=re.sub(digit," ",argStr) 270 argStr=re.sub("\.","",argStr) 271 argStr=re.sub(",","",argStr) 272 argStr=re.sub('\+',' ',argStr) 273 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 274 return argStr.strip() 275 276 def addToSources(self,exfile,rpath,srcDict): 277 """ 278 Put into data structure that allows easy generation of makefile 279 """ 280 pkg=rpath.split(os.path.sep)[0] 281 relpfile=os.path.join(rpath,exfile) 282 lang=self.getLanguage(exfile) 283 if not lang: return 284 self.sources[pkg][lang]['srcs'].append(relpfile) 285 self.sources[pkg][lang][relpfile] = [] 286 if 'depends' in srcDict: 287 depSrcList=srcDict['depends'].split() 288 for depSrc in depSrcList: 289 depObj=os.path.splitext(depSrc)[0]+".o" 290 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 291 292 # In gmakefile, ${TESTDIR} var specifies the object compilation 293 testsdir=rpath+"/" 294 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 295 self.objects[pkg].append(objfile) 296 return 297 298 def addToTests(self,test,rpath,exfile,execname,testDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 Organized by languages to allow testing of languages 302 """ 303 pkg=rpath.split("/")[0] 304 nmtest=os.path.join(rpath,test) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 self.tests[pkg][lang][nmtest]={} 308 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 309 self.tests[pkg][lang][nmtest]['exec']=execname 310 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 311 return 312 313 def getExecname(self,exfile,rpath): 314 """ 315 Generate bash script using template found next to this file. 316 This file is read in at constructor time to avoid file I/O 317 """ 318 if self.single_ex: 319 execname=rpath.split("/")[1]+"-ex" 320 else: 321 execname=os.path.splitext(exfile)[0] 322 return execname 323 324 def getSubstVars(self,testDict,rpath,testname): 325 """ 326 Create a dictionary with all of the variables that get substituted 327 into the template commands found in example_template.py 328 """ 329 subst={} 330 331 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 332 if 'nsize' not in testDict: testDict['nsize']=1 333 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 334 for ak in testparse.acceptedkeys: 335 if ak=='test': continue 336 subst[ak]=(testDict[ak] if ak in testDict else '') 337 338 # Now do other variables 339 subst['execname']=testDict['execname'] 340 if 'filter' in testDict: 341 subst['filter']="'"+testDict['filter']+"'" # Quotes are tricky - overwrite 342 343 # Others 344 subst['subargs']='' # Default. For variables override 345 subst['srcdir']=os.path.join(os.path.dirname(self.srcdir), 'src', rpath) 346 subst['label_suffix']='' 347 subst['comments']="\n#".join(subst['comments'].split("\n")) 348 if subst['comments']: subst['comments']="#"+subst['comments'] 349 subst['exec']="../"+subst['execname'] 350 subst['testroot']=self.testroot_dir 351 subst['testname']=testname 352 dp = self.conf.get('DATAFILESPATH','') 353 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 354 355 # This is used to label some matrices 356 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 357 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 358 359 # These can have for loops and are treated separately later 360 subst['nsize']=str(subst['nsize']) 361 362 #Conf vars 363 if self.petsc_arch.find('valgrind')>=0: 364 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 365 else: 366 subst['mpiexec']=self.conf['MPIEXEC'] 367 subst['petsc_dir']=self.petsc_dir # not self.conf['PETSC_DIR'] as this could be windows path 368 subst['petsc_arch']=self.petsc_arch 369 if self.inInstallDir: 370 # Case 2 371 subst['CONFIG_DIR']=os.path.join(os.path.dirname(self.srcdir),'config') 372 else: 373 # Case 1 374 subst['CONFIG_DIR']=os.path.join(self.petsc_dir,'config') 375 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 376 subst['diff']=self.conf['DIFF'] 377 subst['rm']=self.conf['RM'] 378 subst['grep']=self.conf['GREP'] 379 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 380 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 381 382 # Output file is special because of subtests override 383 defroot=(re.sub("run","",testname) if testname.startswith("run") else testname) 384 if not "_" in defroot: defroot=defroot+"_1" 385 subst['defroot']=defroot 386 subst['label']=self.nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 387 subst['redirect_file']=defroot+".tmp" 388 if 'output_file' not in testDict: 389 subst['output_file']="output/"+defroot+".out" 390 # Add in the full path here. 391 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 392 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 393 if not subst['TODO']: 394 print("Warning: "+subst['output_file']+" not found.") 395 # Worry about alt files here -- see 396 # src/snes/examples/tutorials/output/ex22*.out 397 altlist=[subst['output_file']] 398 basefile,ext = os.path.splitext(subst['output_file']) 399 for i in range(1,9): 400 altroot=basefile+"_alt" 401 if i > 1: altroot=altroot+"_"+str(i) 402 af=altroot+".out" 403 srcaf=os.path.join(subst['srcdir'],af) 404 fullaf=os.path.join(self.petsc_dir,srcaf) 405 if os.path.isfile(fullaf): altlist.append(srcaf) 406 if len(altlist)>1: subst['altfiles']=altlist 407 #if len(altlist)>1: print("Found alt files: ",altlist) 408 409 subst['regexes']={} 410 for subkey in subst: 411 if subkey=='regexes': continue 412 if not isinstance(subst[subkey],str): continue 413 patt="@"+subkey.upper()+"@" 414 subst['regexes'][subkey]=re.compile(patt) 415 416 return subst 417 418 def _substVars(self,subst,origStr): 419 """ 420 Substitute variables 421 """ 422 Str=origStr 423 for subkey in subst: 424 if subkey=='regexes': continue 425 if not isinstance(subst[subkey],str): continue 426 if subkey.upper() not in Str: continue 427 Str=subst['regexes'][subkey].sub(subst[subkey],Str) 428 return Str 429 430 def getCmds(self,subst,i): 431 """ 432 Generate bash script using template found next to this file. 433 This file is read in at constructor time to avoid file I/O 434 """ 435 nindnt=i # the start and has to be consistent with below 436 cmdindnt=self.indent*nindnt 437 cmdLines="" 438 439 # MPI is the default -- but we have a few odd commands 440 if not subst['command']: 441 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 442 else: 443 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 444 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 445 446 cmdLines+=cmdindnt+'if test $res = 0; then\n' 447 diffindnt=self.indent*(nindnt+1) 448 if not subst['filter_output']: 449 if 'altfiles' not in subst: 450 cmd=diffindnt+self._substVars(subst,example_template.difftest) 451 else: 452 # Have to do it by hand a bit because of variable number of alt files 453 rf=subst['redirect_file'] 454 cmd=diffindnt+example_template.difftest.split('@')[0] 455 for i in range(len(subst['altfiles'])): 456 af=subst['altfiles'][i] 457 cmd+=af+' '+rf 458 if i!=len(subst['altfiles'])-1: 459 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 460 cmd+=' || ${diff_exe} ' 461 else: 462 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 463 cmd+=subst['label_suffix']+' ""' # Quotes are painful 464 else: 465 cmd=diffindnt+self._substVars(subst,example_template.filterdifftest) 466 cmdLines+=cmd+"\n" 467 cmdLines+=cmdindnt+'else\n' 468 cmdLines+=diffindnt+'printf "ok ${label} # SKIP Command failed so no diff\\n"\n' 469 cmdLines+=cmdindnt+'fi\n' 470 return cmdLines 471 472 def _writeTodoSkip(self,fh,tors,reasons,footer): 473 """ 474 Write out the TODO and SKIP lines in the file 475 The TODO or SKIP variable, tors, should be lower case 476 """ 477 TORS=tors.upper() 478 template=eval("example_template."+tors+"line") 479 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 480 tab = '' 481 if reasons: 482 fh.write('if ! $force; then\n') 483 tab = tab + ' ' 484 if reasons == ["Requires DATAFILESPATH"]: 485 # The only reason not to run is DATAFILESPATH, which we check at run-time 486 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 487 tab = tab + ' ' 488 if reasons: 489 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 490 fh.write(tab+footer+"\n") 491 fh.write(tab+"exit\n") 492 if reasons == ["Requires DATAFILESPATH"]: 493 fh.write(' fi\n') 494 if reasons: 495 fh.write('fi\n') 496 fh.write('\n\n') 497 return 498 499 def getLoopVarsHead(self,loopVars,i): 500 """ 501 Generate a nicely indented string with the format loops 502 Here is what the data structure looks like 503 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 504 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 505 loopVars['subargs']['pc_type']=["j","cholesky sor"] 506 """ 507 outstr=''; indnt=self.indent 508 for key in loopVars: 509 for var in loopVars[key]['varlist']: 510 varval=loopVars[key][var] 511 outstr += indnt * i + "for "+varval[0]+" in "+varval[1]+"; do\n" 512 i = i + 1 513 return (outstr,i) 514 515 def getLoopVarsFoot(self,loopVars,i): 516 outstr=''; indnt=self.indent 517 for key in loopVars: 518 for var in loopVars[key]['varlist']: 519 i = i - 1 520 outstr += indnt * i + "done\n" 521 return (outstr,i) 522 523 def genRunScript(self,testname,root,isRun,srcDict): 524 """ 525 Generate bash script using template found next to this file. 526 This file is read in at constructor time to avoid file I/O 527 """ 528 # runscript_dir directory has to be consistent with gmakefile 529 testDict=srcDict[testname] 530 rpath=self.srcrelpath(root) 531 runscript_dir=os.path.join(self.testroot_dir,rpath) 532 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 533 fh=open(os.path.join(runscript_dir,testname+".sh"),"w") 534 535 # Get variables to go into shell scripts. last time testDict used 536 subst=self.getSubstVars(testDict,rpath,testname) 537 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 538 539 #Handle runfiles 540 for lfile in subst.get('localrunfiles','').split(): 541 install_files(os.path.join(root, lfile), 542 os.path.join(runscript_dir, os.path.dirname(lfile))) 543 # Check subtests for local runfiles 544 for stest in subst.get("subtests",[]): 545 for lfile in testDict[stest].get('localrunfiles','').split(): 546 install_files(os.path.join(root, lfile), 547 os.path.join(runscript_dir, os.path.dirname(lfile))) 548 549 # Now substitute the key variables into the header and footer 550 header=self._substVars(subst,example_template.header) 551 # The header is done twice to enable @...@ in header 552 header=self._substVars(subst,header) 553 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 554 555 # Start writing the file 556 fh.write(header+"\n") 557 558 # If there is a TODO or a SKIP then we do it before writing out the 559 # rest of the command (which is useful for working on the test) 560 # SKIP and TODO can be for the source file or for the runs 561 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 562 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 563 564 j=0 # for indentation 565 566 if loopVars: 567 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 568 if (loopHead): fh.write(loopHead+"\n") 569 570 # Subtests are special 571 if 'subtests' in testDict: 572 substP=subst # Subtests can inherit args but be careful 573 k=0 # for label suffixes 574 for stest in testDict["subtests"]: 575 subst=substP.copy() 576 subst.update(testDict[stest]) 577 # nsize is special because it is usually overwritten 578 if 'nsize' in testDict[stest]: 579 fh.write("nsize="+str(testDict[stest]['nsize'])+"\n") 580 else: 581 fh.write("nsize=1\n") 582 subst['label_suffix']='-'+string.ascii_letters[k]; k+=1 583 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 584 if sLoopVars: 585 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j) 586 fh.write(sLoopHead+"\n") 587 fh.write(self.getCmds(subst,j)+"\n") 588 if sLoopVars: 589 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 590 fh.write(sLoopFoot+"\n") 591 else: 592 fh.write(self.getCmds(subst,j)+"\n") 593 594 if loopVars: 595 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 596 fh.write(loopFoot+"\n") 597 598 fh.write(footer+"\n") 599 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 600 #if '10_9' in testname: sys.exit() 601 return 602 603 def genScriptsAndInfo(self,exfile,root,srcDict): 604 """ 605 Generate scripts from the source file, determine if built, etc. 606 For every test in the exfile with info in the srcDict: 607 1. Determine if it needs to be run for this arch 608 2. Generate the script 609 3. Generate the data needed to write out the makefile in a 610 convenient way 611 All tests are *always* run, but some may be SKIP'd per the TAP standard 612 """ 613 debug=False 614 rpath=self.srcrelpath(root) 615 execname=self.getExecname(exfile,rpath) 616 isBuilt=self._isBuilt(exfile,srcDict) 617 for test in srcDict: 618 if test in self.buildkeys: continue 619 if debug: print(self.nameSpace(exfile,root), test) 620 srcDict[test]['execname']=execname # Convenience in generating scripts 621 isRun=self._isRun(srcDict[test]) 622 self.genRunScript(test,root,isRun,srcDict) 623 srcDict[test]['isrun']=isRun 624 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 625 626 # This adds to datastructure for building deps 627 if isBuilt: self.addToSources(exfile,rpath,srcDict) 628 return 629 630 def _isBuilt(self,exfile,srcDict): 631 """ 632 Determine if this file should be built. 633 """ 634 # Get the language based on file extension 635 srcDict['SKIP'] = [] 636 lang=self.getLanguage(exfile) 637 if (lang=="F" or lang=="F90"): 638 if not self.have_fortran: 639 srcDict["SKIP"].append("Fortran required for this test") 640 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 641 srcDict["SKIP"].append("Fortran f90freeform required for this test") 642 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 643 srcDict["SKIP"].append("CUDA required for this test") 644 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 645 srcDict["SKIP"].append("C++ required for this test") 646 647 # Deprecated source files 648 if srcDict.get("TODO"): 649 return False 650 651 # isRun can work with srcDict to handle the requires 652 if "requires" in srcDict: 653 if srcDict["requires"]: 654 return self._isRun(srcDict) 655 656 return srcDict['SKIP'] == [] 657 658 659 def _isRun(self,testDict, debug=False): 660 """ 661 Based on the requirements listed in the src file and the petscconf.h 662 info, determine whether this test should be run or not. 663 """ 664 indent=" " 665 666 if 'SKIP' not in testDict: 667 testDict['SKIP'] = [] 668 # MPI requirements 669 if 'MPI_IS_MPIUNI' in self.conf: 670 nsize=testDict.get('nsize','1') 671 if str(nsize) != '1': 672 testDict['SKIP'].append("Parallel test with serial build") 673 674 # The requirements for the test are the sum of all the run subtests 675 if 'subtests' in testDict: 676 if 'requires' not in testDict: testDict['requires']="" 677 for stest in testDict['subtests']: 678 if 'requires' in testDict[stest]: 679 testDict['requires']+=" "+testDict[stest]['requires'] 680 nsize=testDict[stest].get('nsize','1') 681 if str(nsize) != '1': 682 testDict['SKIP'].append("Parallel test with serial build") 683 break 684 685 # Now go through all requirements 686 if 'requires' in testDict: 687 for requirement in testDict['requires'].split(): 688 requirement=requirement.strip() 689 if not requirement: continue 690 if debug: print(indent+"Requirement: ", requirement) 691 isNull=False 692 if requirement.startswith("!"): 693 requirement=requirement[1:]; isNull=True 694 # Precision requirement for reals 695 if requirement in self.precision_types: 696 if self.conf['PETSC_PRECISION']==requirement: 697 if isNull: 698 testDict['SKIP'].append("not "+requirement+" required") 699 continue 700 continue # Success 701 elif not isNull: 702 testDict['SKIP'].append(requirement+" required") 703 continue 704 # Precision requirement for ints 705 if requirement in self.integer_types: 706 if requirement=="int32": 707 if self.conf['PETSC_SIZEOF_INT']==4: 708 if isNull: 709 testDict['SKIP'].append("not int32 required") 710 continue 711 continue # Success 712 elif not isNull: 713 testDict['SKIP'].append("int32 required") 714 continue 715 if requirement=="int64": 716 if self.conf['PETSC_SIZEOF_INT']==8: 717 if isNull: 718 testDict['SKIP'].append("NOT int64 required") 719 continue 720 continue # Success 721 elif not isNull: 722 testDict['SKIP'].append("int64 required") 723 continue 724 # Datafilespath 725 if requirement=="datafilespath" and not isNull: 726 testDict['SKIP'].append("Requires DATAFILESPATH") 727 continue 728 # Defines -- not sure I have comments matching 729 if "define(" in requirement.lower(): 730 reqdef=requirement.split("(")[1].split(")")[0] 731 if reqdef in self.conf: 732 if isNull: 733 testDict['SKIP'].append("Null requirement not met: "+requirement) 734 continue 735 continue # Success 736 elif not isNull: 737 testDict['SKIP'].append("Required: "+requirement) 738 continue 739 740 # Rest should be packages that we can just get from conf 741 if requirement == "complex": 742 petscconfvar="PETSC_USE_COMPLEX" 743 else: 744 petscconfvar="PETSC_HAVE_"+requirement.upper() 745 if self.conf.get(petscconfvar): 746 if isNull: 747 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 748 continue 749 continue # Success 750 elif not isNull: 751 if debug: print("requirement not found: ", requirement) 752 testDict['SKIP'].append(petscconfvar+" requirement not met") 753 continue 754 755 return testDict['SKIP'] == [] 756 757 def genPetscTests_summarize(self,dataDict): 758 """ 759 Required method to state what happened 760 """ 761 if not self.summarize: return 762 indent=" " 763 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 764 fh=open(fhname,"w") 765 for root in dataDict: 766 relroot=self.srcrelpath(root) 767 pkg=relroot.split("/")[1] 768 fh.write(relroot+"\n") 769 allSrcs=[] 770 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 771 for exfile in dataDict[root]: 772 # Basic information 773 rfile=os.path.join(relroot,exfile) 774 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 775 fh.write(indent+exfile+indent*4+builtStatus+"\n") 776 777 for test in dataDict[root][exfile]: 778 if test in self.buildkeys: continue 779 line=indent*2+test 780 fh.write(line+"\n") 781 # Looks nice to have the keys in order 782 #for key in dataDict[root][exfile][test]: 783 for key in "isrun abstracted nsize args requires script".split(): 784 if key not in dataDict[root][exfile][test]: continue 785 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 786 fh.write(line+"\n") 787 fh.write("\n") 788 fh.write("\n") 789 fh.write("\n") 790 #fh.write("\nClass Sources\n"+str(self.sources)+"\n") 791 #fh.write("\nClass Tests\n"+str(self.tests)+"\n") 792 fh.close() 793 return 794 795 def genPetscTests(self,root,dirs,files,dataDict): 796 """ 797 Go through and parse the source files in the directory to generate 798 the examples based on the metadata contained in the source files 799 """ 800 debug=False 801 # Use examplesAnalyze to get what the makefles think are sources 802 #self.examplesAnalyze(root,dirs,files,anlzDict) 803 804 dataDict[root]={} 805 806 for exfile in files: 807 #TST: Until we replace files, still leaving the orginals as is 808 #if not exfile.startswith("new_"+"ex"): continue 809 #if not exfile.startswith("ex"): continue 810 811 # Ignore emacs and other temporary files 812 if exfile.startswith("."): continue 813 if exfile.startswith("#"): continue 814 if exfile.endswith("~"): continue 815 # Only parse source files 816 ext=os.path.splitext(exfile)[-1].lstrip('.') 817 if ext not in LANGS: continue 818 819 # Convenience 820 fullex=os.path.join(root,exfile) 821 if self.verbose: print(' --> '+fullex) 822 dataDict[root].update(testparse.parseTestFile(fullex,0)) 823 if exfile in dataDict[root]: 824 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 825 826 return 827 828 def walktree(self,top): 829 """ 830 Walk a directory tree, starting from 'top' 831 """ 832 # Goal of action is to fill this dictionary 833 dataDict={} 834 for root, dirs, files in os.walk(top, topdown=True): 835 dirs.sort() 836 files.sort() 837 if not "examples" in root: continue 838 if "dSYM" in root: continue 839 if os.path.basename(root.rstrip("/")) == 'output': continue 840 if self.verbose: print(root) 841 self.genPetscTests(root,dirs,files,dataDict) 842 # Now summarize this dictionary 843 if self.verbose: self.genPetscTests_summarize(dataDict) 844 return dataDict 845 846 def gen_gnumake(self, fd): 847 """ 848 Overwrite of the method in the base PETSc class 849 """ 850 def write(stem, srcs): 851 for lang in LANGS: 852 if srcs[lang]['srcs']: 853 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 854 for pkg in PKGS: 855 srcs = self.gen_pkg(pkg) 856 write('testsrcs-' + pkg, srcs) 857 # Handle dependencies 858 for lang in LANGS: 859 for exfile in srcs[lang]['srcs']: 860 if exfile in srcs[lang]: 861 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 862 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 863 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 864 if deps: 865 # The executable literally depends on the object file because it is linked 866 fd.write(ex +": " + " ".join(deps) +'\n') 867 # The object file containing 'main' does not normally depend on other object 868 # files, but it does when it includes their modules. This dependency is 869 # overly blunt and could be reduced to only depend on object files for 870 # modules that are used, like "*f90aux.o". 871 fd.write(exfo +": " + " ".join(deps) +'\n') 872 873 return self.gendeps 874 875 def gen_pkg(self, pkg): 876 """ 877 Overwrite of the method in the base PETSc class 878 """ 879 return self.sources[pkg] 880 881 def write_gnumake(self, dataDict, output=None): 882 """ 883 Write out something similar to files from gmakegen.py 884 885 Test depends on script which also depends on source 886 file, but since I don't have a good way generating 887 acting on a single file (oops) just depend on 888 executable which in turn will depend on src file 889 """ 890 # Different options for how to set up the targets 891 compileExecsFirst=False 892 893 # Open file 894 fd = open(output, 'w') 895 896 # Write out the sources 897 gendeps = self.gen_gnumake(fd) 898 899 # Write out the tests and execname targets 900 fd.write("\n#Tests and executables\n") # Delimiter 901 902 for pkg in PKGS: 903 # These grab the ones that are built 904 for lang in LANGS: 905 testdeps=[] 906 for ftest in self.tests[pkg][lang]: 907 test=os.path.basename(ftest) 908 basedir=os.path.dirname(ftest) 909 testdeps.append(self.nameSpace(test,basedir)) 910 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 911 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 912 913 # test targets 914 for ftest in self.tests[pkg][lang]: 915 test=os.path.basename(ftest) 916 basedir=os.path.dirname(ftest) 917 testdir="${TESTDIR}/"+basedir+"/" 918 nmtest=self.nameSpace(test,basedir) 919 rundir=os.path.join(testdir,test) 920 script=test+".sh" 921 922 # Deps 923 exfile=self.tests[pkg][lang][ftest]['exfile'] 924 fullex=os.path.join(self.srcdir,exfile) 925 localexec=self.tests[pkg][lang][ftest]['exec'] 926 execname=os.path.join(testdir,localexec) 927 fullscript=os.path.join(testdir,script) 928 tmpfile=os.path.join(testdir,test,test+".tmp") 929 930 # *.counts depends on the script and either executable (will 931 # be run) or the example source file (SKIP or TODO) 932 fd.write('%s.counts : %s %s' 933 % (os.path.join('$(TESTDIR)/counts', nmtest), 934 fullscript, 935 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 936 ) 937 if exfile in self.sources[pkg][lang]: 938 for dep in self.sources[pkg][lang][exfile]: 939 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 940 fd.write('\n') 941 942 # Now write the args: 943 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 944 945 fd.close() 946 return 947 948def main(petsc_dir=None, petsc_arch=None, verbose=False, single_ex=False, srcdir=None, testdir=None): 949 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 950 if petsc_arch: 951 if len(petsc_arch.split(os.path.sep))>1: 952 petsc_dir,petsc_arch=os.path.split(petsc_arch.rstrip(os.path.sep)) 953 output = os.path.join(testdir, 'testfiles') 954 955 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 956 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 957 testdir=testdir) 958 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 959 pEx.write_gnumake(dataDict, output) 960 961if __name__ == '__main__': 962 import optparse 963 parser = optparse.OptionParser() 964 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 965 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 966 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 967 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 968 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 969 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 970 971 opts, extra_args = parser.parse_args() 972 if extra_args: 973 import sys 974 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 975 exit(1) 976 if opts.testdir is None: 977 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 978 979 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 980 verbose=opts.verbose, 981 single_ex=opts.single_executable, srcdir=opts.srcdir, 982 testdir=opts.testdir) 983