1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 if self.inInstallDir: 103 # Case 2 discussed above 104 # set PETSC_ARCH to install directory to allow script to work in both 105 dirlist=thisscriptdir.split(os.path.sep) 106 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 107 self.arch_dir=installdir 108 if self.srcdir is None: 109 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 110 else: 111 if petsc_arch == '': 112 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 113 # Case 1 discussed above 114 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 115 if self.srcdir is None: 116 self.srcdir=os.path.join(self.petsc_dir,'src') 117 118 self.testroot_dir=os.path.abspath(testdir) 119 120 self.verbose=verbose 121 # Whether to write out a useful debugging 122 self.summarize=True if verbose else False 123 124 # For help in setting the requirements 125 self.precision_types="single double __float128 int32".split() 126 self.integer_types="int32 int64 long32 long64".split() 127 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 128 129 # Things that are not test 130 self.buildkeys=testparse.buildkeys 131 132 # Adding a dictionary for storing sources, objects, and tests 133 # to make building the dependency tree easier 134 self.sources={} 135 self.objects={} 136 self.tests={} 137 for pkg in self.pkg_pkgs: 138 self.sources[pkg]={} 139 self.objects[pkg]=[] 140 self.tests[pkg]={} 141 for lang in LANGS: 142 self.sources[pkg][lang]={} 143 self.sources[pkg][lang]['srcs']=[] 144 self.tests[pkg][lang]={} 145 146 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 147 148 self.indent=" " 149 if self.verbose: print('Finishing the constructor') 150 return 151 152 def srcrelpath(self,rdir): 153 """ 154 Get relative path to source directory 155 """ 156 return os.path.relpath(rdir,self.srcdir) 157 158 def getInInstallDir(self,thisscriptdir): 159 """ 160 When petsc is installed then this file in installed in: 161 <PREFIX>/share/petsc/examples/config/gmakegentest.py 162 otherwise the path is: 163 <PETSC_DIR>/config/gmakegentest.py 164 We use this difference to determine if we are in installdir 165 """ 166 dirlist=thisscriptdir.split(os.path.sep) 167 if len(dirlist)>4: 168 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 169 if lastfour==os.path.join('share','petsc','examples','config'): 170 return True 171 else: 172 return False 173 else: 174 return False 175 176 def getLanguage(self,srcfile): 177 """ 178 Based on the source, determine associated language as found in gmakegen.LANGS 179 Can we just return srcext[1:\] now? 180 """ 181 langReq=None 182 srcext=os.path.splitext(srcfile)[-1] 183 if srcext in ".F90".split(): langReq="F90" 184 if srcext in ".F".split(): langReq="F" 185 if srcext in ".cxx".split(): langReq="cxx" 186 if srcext in ".cpp".split(): langReq="cpp" 187 if srcext == ".cu": langReq="cu" 188 if srcext == ".c": langReq="c" 189 #if not langReq: print("ERROR: ", srcext, srcfile) 190 return langReq 191 192 def _getAltList(self,output_file,srcdir): 193 ''' Calculate AltList based on output file-- see 194 src/snes/tutorials/output/ex22*.out 195 ''' 196 altlist=[output_file] 197 basefile,ext = os.path.splitext(output_file) 198 for i in range(1,9): 199 altroot=basefile+"_alt" 200 if i > 1: altroot=altroot+"_"+str(i) 201 af=altroot+".out" 202 srcaf=os.path.join(srcdir,af) 203 fullaf=os.path.join(self.petsc_dir,srcaf) 204 if os.path.isfile(fullaf): altlist.append(srcaf) 205 206 return altlist 207 208 209 def _getLoopVars(self,inDict,testname, isSubtest=False): 210 """ 211 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 212 Return: 213 inDict['args']: -ksp_monitor 214 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 215 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 216 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 217 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 218 subst should be passed in instead of inDict 219 """ 220 loopVars={}; newargs=[] 221 lsuffix='+' 222 argregex = re.compile(' (?=-[a-zA-Z])') 223 from testparse import parseLoopArgs 224 for key in inDict: 225 if key in ('SKIP', 'regexes'): 226 continue 227 akey=('subargs' if key=='args' else key) # what to assign 228 if akey not in inDict: inDict[akey]='' 229 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 230 # Always generate a loop over nsize, even if there is only one value 231 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 232 keystr = str(inDict[key]) 233 varlist = [] 234 for varset in argregex.split(keystr): 235 if not varset.strip(): continue 236 if '{{' in varset: 237 keyvar,lvars,ftype=parseLoopArgs(varset) 238 if akey not in loopVars: loopVars[akey]={} 239 varlist.append(keyvar) 240 loopVars[akey][keyvar]=[keyvar,lvars] 241 if akey=='nsize': 242 if len(lvars.split()) > 1: 243 lsuffix += akey +'-${i' + keyvar + '}' 244 else: 245 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 246 lsuffix+=keyvar+'-${i' + keyvar + '}_' 247 else: 248 if key=='args': 249 newargs.append(varset.strip()) 250 if varlist: 251 loopVars[akey]['varlist']=varlist 252 253 # For subtests, args are always substituted in (not top level) 254 if isSubtest: 255 inDict['subargs'] += " "+" ".join(newargs) 256 inDict['args']='' 257 if 'label_suffix' in inDict: 258 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 259 else: 260 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 261 else: 262 if loopVars: 263 inDict['args'] = ' '.join(newargs) 264 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 265 return loopVars 266 267 def getArgLabel(self,testDict): 268 """ 269 In all of the arguments in the test dictionary, create a simple 270 string for searching within the makefile system. For simplicity in 271 search, remove "-", for strings, etc. 272 Also, concatenate the arg commands 273 For now, ignore nsize -- seems hard to search for anyway 274 """ 275 # Collect all of the args associated with a test 276 argStr=("" if 'args' not in testDict else testDict['args']) 277 if 'subtests' in testDict: 278 for stest in testDict["subtests"]: 279 sd=testDict[stest] 280 argStr=argStr+("" if 'args' not in sd else sd['args']) 281 282 # Now go through and cleanup 283 argStr=re.sub('{{(.*?)}}',"",argStr) 284 argStr=re.sub('-'," ",argStr) 285 for digit in string.digits: argStr=re.sub(digit," ",argStr) 286 argStr=re.sub("\.","",argStr) 287 argStr=re.sub(",","",argStr) 288 argStr=re.sub('\+',' ',argStr) 289 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 290 return argStr.strip() 291 292 def addToSources(self,exfile,rpath,srcDict): 293 """ 294 Put into data structure that allows easy generation of makefile 295 """ 296 pkg=rpath.split(os.path.sep)[0] 297 relpfile=os.path.join(rpath,exfile) 298 lang=self.getLanguage(exfile) 299 if not lang: return 300 if pkg not in self.sources: return 301 self.sources[pkg][lang]['srcs'].append(relpfile) 302 self.sources[pkg][lang][relpfile] = [] 303 if 'depends' in srcDict: 304 depSrcList=srcDict['depends'].split() 305 for depSrc in depSrcList: 306 depObj=os.path.splitext(depSrc)[0]+".o" 307 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 308 309 # In gmakefile, ${TESTDIR} var specifies the object compilation 310 testsdir=rpath+"/" 311 objfile="${TESTDIR}/"+testsdir+os.path.splitext(exfile)[0]+".o" 312 self.objects[pkg].append(objfile) 313 return 314 315 def addToTests(self,test,rpath,exfile,execname,testDict): 316 """ 317 Put into data structure that allows easy generation of makefile 318 Organized by languages to allow testing of languages 319 """ 320 pkg=rpath.split("/")[0] 321 nmtest=os.path.join(rpath,test) 322 lang=self.getLanguage(exfile) 323 if not lang: return 324 if pkg not in self.tests: return 325 self.tests[pkg][lang][nmtest]={} 326 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 327 self.tests[pkg][lang][nmtest]['exec']=execname 328 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 329 return 330 331 def getExecname(self,exfile,rpath): 332 """ 333 Generate bash script using template found next to this file. 334 This file is read in at constructor time to avoid file I/O 335 """ 336 if self.single_ex: 337 execname=rpath.split("/")[1]+"-ex" 338 else: 339 execname=os.path.splitext(exfile)[0] 340 return execname 341 342 def getSubstVars(self,testDict,rpath,testname): 343 """ 344 Create a dictionary with all of the variables that get substituted 345 into the template commands found in example_template.py 346 """ 347 subst={} 348 349 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 350 if 'nsize' not in testDict: testDict['nsize'] = '1' 351 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 352 for ak in testparse.acceptedkeys: 353 if ak=='test': continue 354 subst[ak]=(testDict[ak] if ak in testDict else '') 355 356 # Now do other variables 357 subst['execname']=testDict['execname'] 358 subst['error']='' 359 if 'filter' in testDict: 360 if testDict['filter'].startswith("Error:"): 361 subst['error']="Error" 362 subst['filter']=testDict['filter'].lstrip("Error:") 363 else: 364 subst['filter']=testDict['filter'] 365 366 # Others 367 subst['subargs']='' # Default. For variables override 368 subst['srcdir']=os.path.join(self.srcdir, rpath) 369 subst['label_suffix']='' 370 subst['comments']="\n#".join(subst['comments'].split("\n")) 371 if subst['comments']: subst['comments']="#"+subst['comments'] 372 subst['exec']="../"+subst['execname'] 373 subst['testroot']=self.testroot_dir 374 subst['testname']=testname 375 dp = self.conf.get('DATAFILESPATH','') 376 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 377 378 # This is used to label some matrices 379 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 380 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 381 382 #Conf vars 383 if self.petsc_arch.find('valgrind')>=0: 384 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 385 else: 386 subst['mpiexec']=self.conf['MPIEXEC'] 387 subst['pkg_name']=self.pkg_name 388 subst['pkg_dir']=self.pkg_dir 389 subst['pkg_arch']=self.petsc_arch 390 subst['CONFIG_DIR']=thisscriptdir 391 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 392 subst['diff']=self.conf['DIFF'] 393 subst['rm']=self.conf['RM'] 394 subst['grep']=self.conf['GREP'] 395 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 396 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 397 398 # Output file is special because of subtests override 399 defroot = testparse.getDefaultOutputFileRoot(testname) 400 if 'output_file' not in testDict: 401 subst['output_file']="output/"+defroot+".out" 402 subst['redirect_file']=defroot+".tmp" 403 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 404 405 # Add in the full path here. 406 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 407 408 subst['regexes']={} 409 for subkey in subst: 410 if subkey=='regexes': continue 411 if not isinstance(subst[subkey],str): continue 412 patt="@"+subkey.upper()+"@" 413 subst['regexes'][subkey]=re.compile(patt) 414 415 return subst 416 417 def _substVars(self,subst,origStr): 418 """ 419 Substitute variables 420 """ 421 Str=origStr 422 for subkey in subst: 423 if subkey=='regexes': continue 424 if not isinstance(subst[subkey],str): continue 425 if subkey.upper() not in Str: continue 426 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 427 return Str 428 429 def getCmds(self,subst,i, debug=False): 430 """ 431 Generate bash script using template found next to this file. 432 This file is read in at constructor time to avoid file I/O 433 """ 434 nindnt=i # the start and has to be consistent with below 435 cmdindnt=self.indent*nindnt 436 cmdLines="" 437 438 # MPI is the default -- but we have a few odd commands 439 if not subst['command']: 440 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 441 else: 442 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 443 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 444 445 cmdLines+=cmdindnt+'if test $res = 0; then\n' 446 diffindnt=self.indent*(nindnt+1) 447 448 # Do some checks on existence of output_file and alt files 449 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 450 if not subst['TODO']: 451 print("Warning: "+subst['output_file']+" not found.") 452 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 453 454 # altlist always has output_file 455 if len(altlist)==1: 456 cmd=diffindnt+self._substVars(subst,example_template.difftest) 457 else: 458 if debug: print("Found alt files: ",altlist) 459 # Have to do it by hand a bit because of variable number of alt files 460 rf=subst['redirect_file'] 461 cmd=diffindnt+example_template.difftest.split('@')[0] 462 for i in range(len(altlist)): 463 af=altlist[i] 464 cmd+=af+' '+rf 465 if i!=len(altlist)-1: 466 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 467 cmd+=' || ${diff_exe} ' 468 else: 469 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 470 cmd+=subst['label_suffix']+' ""' # Quotes are painful 471 cmdLines+=cmd+"\n" 472 cmdLines+=cmdindnt+'else\n' 473 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 474 cmdLines+=cmdindnt+'fi\n' 475 return cmdLines 476 477 def _writeTodoSkip(self,fh,tors,reasons,footer): 478 """ 479 Write out the TODO and SKIP lines in the file 480 The TODO or SKIP variable, tors, should be lower case 481 """ 482 TORS=tors.upper() 483 template=eval("example_template."+tors+"line") 484 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 485 tab = '' 486 if reasons: 487 fh.write('if ! $force; then\n') 488 tab = tab + ' ' 489 if reasons == ["Requires DATAFILESPATH"]: 490 # The only reason not to run is DATAFILESPATH, which we check at run-time 491 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 492 tab = tab + ' ' 493 if reasons: 494 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 495 fh.write(tab+footer+"\n") 496 fh.write(tab+"exit\n") 497 if reasons == ["Requires DATAFILESPATH"]: 498 fh.write(' fi\n') 499 if reasons: 500 fh.write('fi\n') 501 fh.write('\n\n') 502 return 503 504 def getLoopVarsHead(self,loopVars,i,usedVars={}): 505 """ 506 Generate a nicely indented string with the format loops 507 Here is what the data structure looks like 508 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 509 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 510 loopVars['subargs']['pc_type']=["j","cholesky sor"] 511 """ 512 outstr=''; indnt=self.indent 513 514 for key in loopVars: 515 for var in loopVars[key]['varlist']: 516 varval=loopVars[key][var] 517 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 518 outstr += "\n\n" 519 520 for key in loopVars: 521 for var in loopVars[key]['varlist']: 522 varval=loopVars[key][var] 523 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 524 i = i + 1 525 return (outstr,i) 526 527 def getLoopVarsFoot(self,loopVars,i): 528 outstr=''; indnt=self.indent 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 i = i - 1 532 outstr += indnt * i + "done\n" 533 return (outstr,i) 534 535 def genRunScript(self,testname,root,isRun,srcDict): 536 """ 537 Generate bash script using template found next to this file. 538 This file is read in at constructor time to avoid file I/O 539 """ 540 # runscript_dir directory has to be consistent with gmakefile 541 testDict=srcDict[testname] 542 rpath=self.srcrelpath(root) 543 runscript_dir=os.path.join(self.testroot_dir,rpath) 544 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 545 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 546 547 # Get variables to go into shell scripts. last time testDict used 548 subst=self.getSubstVars(testDict,rpath,testname) 549 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 550 if 'subtests' in testDict: 551 # The subtests inherit inDict, so we don't need top-level loops. 552 loopVars = {} 553 554 #Handle runfiles 555 for lfile in subst.get('localrunfiles','').split(): 556 install_files(os.path.join(root, lfile), 557 os.path.join(runscript_dir, os.path.dirname(lfile))) 558 # Check subtests for local runfiles 559 for stest in subst.get("subtests",[]): 560 for lfile in testDict[stest].get('localrunfiles','').split(): 561 install_files(os.path.join(root, lfile), 562 os.path.join(runscript_dir, os.path.dirname(lfile))) 563 564 # Now substitute the key variables into the header and footer 565 header=self._substVars(subst,example_template.header) 566 # The header is done twice to enable @...@ in header 567 header=self._substVars(subst,header) 568 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 569 570 # Start writing the file 571 fh.write(header+"\n") 572 573 # If there is a TODO or a SKIP then we do it before writing out the 574 # rest of the command (which is useful for working on the test) 575 # SKIP and TODO can be for the source file or for the runs 576 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 577 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 578 579 j=0 # for indentation 580 581 if loopVars: 582 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 583 if (loopHead): fh.write(loopHead+"\n") 584 585 # Subtests are special 586 allLoopVars=list(loopVars.keys()) 587 if 'subtests' in testDict: 588 substP=subst # Subtests can inherit args but be careful 589 k=0 # for label suffixes 590 for stest in testDict["subtests"]: 591 subst=substP.copy() 592 subst.update(testDict[stest]) 593 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 594 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 595 if sLoopVars: 596 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 597 allLoopVars+=list(sLoopVars.keys()) 598 fh.write(sLoopHead+"\n") 599 fh.write(self.getCmds(subst,j)+"\n") 600 if sLoopVars: 601 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 602 fh.write(sLoopFoot+"\n") 603 else: 604 fh.write(self.getCmds(subst,j)+"\n") 605 606 if loopVars: 607 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 608 fh.write(loopFoot+"\n") 609 610 fh.write(footer+"\n") 611 612 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 613 #if '10_9' in testname: sys.exit() 614 return 615 616 def genScriptsAndInfo(self,exfile,root,srcDict): 617 """ 618 Generate scripts from the source file, determine if built, etc. 619 For every test in the exfile with info in the srcDict: 620 1. Determine if it needs to be run for this arch 621 2. Generate the script 622 3. Generate the data needed to write out the makefile in a 623 convenient way 624 All tests are *always* run, but some may be SKIP'd per the TAP standard 625 """ 626 debug=False 627 rpath=self.srcrelpath(root) 628 execname=self.getExecname(exfile,rpath) 629 isBuilt=self._isBuilt(exfile,srcDict) 630 for test in srcDict: 631 if test in self.buildkeys: continue 632 if debug: print(nameSpace(exfile,root), test) 633 srcDict[test]['execname']=execname # Convenience in generating scripts 634 isRun=self._isRun(srcDict[test]) 635 self.genRunScript(test,root,isRun,srcDict) 636 srcDict[test]['isrun']=isRun 637 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 638 639 # This adds to datastructure for building deps 640 if isBuilt: self.addToSources(exfile,rpath,srcDict) 641 return 642 643 def _isBuilt(self,exfile,srcDict): 644 """ 645 Determine if this file should be built. 646 """ 647 # Get the language based on file extension 648 srcDict['SKIP'] = [] 649 lang=self.getLanguage(exfile) 650 if (lang=="F" or lang=="F90"): 651 if not self.have_fortran: 652 srcDict["SKIP"].append("Fortran required for this test") 653 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 654 srcDict["SKIP"].append("Fortran f90freeform required for this test") 655 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 656 srcDict["SKIP"].append("CUDA required for this test") 657 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 658 srcDict["SKIP"].append("HIP required for this test") 659 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 660 srcDict["SKIP"].append("SYCL required for this test") 661 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 662 srcDict["SKIP"].append("C++ required for this test") 663 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 664 srcDict["SKIP"].append("C++ required for this test") 665 666 # Deprecated source files 667 if srcDict.get("TODO"): 668 return False 669 670 # isRun can work with srcDict to handle the requires 671 if "requires" in srcDict: 672 if srcDict["requires"]: 673 return self._isRun(srcDict) 674 675 return srcDict['SKIP'] == [] 676 677 678 def _isRun(self,testDict, debug=False): 679 """ 680 Based on the requirements listed in the src file and the petscconf.h 681 info, determine whether this test should be run or not. 682 """ 683 indent=" " 684 685 if 'SKIP' not in testDict: 686 testDict['SKIP'] = [] 687 # MPI requirements 688 if 'MPI_IS_MPIUNI' in self.conf: 689 if testDict.get('nsize', '1') != '1': 690 testDict['SKIP'].append("Parallel test with serial build") 691 692 # The requirements for the test are the sum of all the run subtests 693 if 'subtests' in testDict: 694 if 'requires' not in testDict: testDict['requires']="" 695 for stest in testDict['subtests']: 696 if 'requires' in testDict[stest]: 697 testDict['requires']+=" "+testDict[stest]['requires'] 698 if testDict[stest].get('nsize', '1') != '1': 699 testDict['SKIP'].append("Parallel test with serial build") 700 break 701 702 # Now go through all requirements 703 if 'requires' in testDict: 704 for requirement in testDict['requires'].split(): 705 requirement=requirement.strip() 706 if not requirement: continue 707 if debug: print(indent+"Requirement: ", requirement) 708 isNull=False 709 if requirement.startswith("!"): 710 requirement=requirement[1:]; isNull=True 711 # Precision requirement for reals 712 if requirement in self.precision_types: 713 if self.conf['PETSC_PRECISION']==requirement: 714 if isNull: 715 testDict['SKIP'].append("not "+requirement+" required") 716 continue 717 continue # Success 718 elif not isNull: 719 testDict['SKIP'].append(requirement+" required") 720 continue 721 # Precision requirement for ints 722 if requirement in self.integer_types: 723 if requirement=="int32": 724 if self.conf['PETSC_SIZEOF_INT']==4: 725 if isNull: 726 testDict['SKIP'].append("not int32 required") 727 continue 728 continue # Success 729 elif not isNull: 730 testDict['SKIP'].append("int32 required") 731 continue 732 if requirement=="int64": 733 if self.conf['PETSC_SIZEOF_INT']==8: 734 if isNull: 735 testDict['SKIP'].append("NOT int64 required") 736 continue 737 continue # Success 738 elif not isNull: 739 testDict['SKIP'].append("int64 required") 740 continue 741 if requirement.startswith("long"): 742 reqsize = int(requirement[4:])//8 743 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 744 if longsize==reqsize: 745 if isNull: 746 testDict['SKIP'].append("not %s required" % requirement) 747 continue 748 continue # Success 749 elif not isNull: 750 testDict['SKIP'].append("%s required" % requirement) 751 continue 752 # Datafilespath 753 if requirement=="datafilespath" and not isNull: 754 testDict['SKIP'].append("Requires DATAFILESPATH") 755 continue 756 # Defines -- not sure I have comments matching 757 if "define(" in requirement.lower(): 758 reqdef=requirement.split("(")[1].split(")")[0] 759 if reqdef in self.conf: 760 if isNull: 761 testDict['SKIP'].append("Null requirement not met: "+requirement) 762 continue 763 continue # Success 764 elif not isNull: 765 testDict['SKIP'].append("Required: "+requirement) 766 continue 767 768 # Rest should be packages that we can just get from conf 769 if requirement in ["complex","debug"]: 770 petscconfvar="PETSC_USE_"+requirement.upper() 771 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 772 else: 773 petscconfvar="PETSC_HAVE_"+requirement.upper() 774 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 775 petsccv = self.conf.get(petscconfvar) 776 pkgcv = self.conf.get(pkgconfvar) 777 778 if petsccv or pkgcv: 779 if isNull: 780 if petsccv: 781 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 782 continue 783 else: 784 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 785 continue 786 continue # Success 787 elif not isNull: 788 if not petsccv and not pkgcv: 789 if debug: print("requirement not found: ", requirement) 790 if self.pkg_name == 'petsc': 791 testDict['SKIP'].append(petscconfvar+" requirement not met") 792 else: 793 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 794 continue 795 return testDict['SKIP'] == [] 796 797 def checkOutput(self,exfile,root,srcDict): 798 """ 799 Check and make sure the output files are in the output director 800 """ 801 debug=False 802 rpath=self.srcrelpath(root) 803 for test in srcDict: 804 if test in self.buildkeys: continue 805 if debug: print(rpath, exfile, test) 806 if 'output_file' in srcDict[test]: 807 output_file=srcDict[test]['output_file'] 808 else: 809 defroot = testparse.getDefaultOutputFileRoot(test) 810 if 'TODO' in srcDict[test]: continue 811 output_file="output/"+defroot+".out" 812 813 fullout=os.path.join(root,output_file) 814 if debug: print("---> ",fullout) 815 if not os.path.exists(fullout): 816 self.missing_files.append(fullout) 817 818 return 819 820 def genPetscTests_summarize(self,dataDict): 821 """ 822 Required method to state what happened 823 """ 824 if not self.summarize: return 825 indent=" " 826 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 827 with open(fhname, "w") as fh: 828 for root in dataDict: 829 relroot=self.srcrelpath(root) 830 pkg=relroot.split("/")[1] 831 fh.write(relroot+"\n") 832 allSrcs=[] 833 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 834 for exfile in dataDict[root]: 835 # Basic information 836 rfile=os.path.join(relroot,exfile) 837 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 838 fh.write(indent+exfile+indent*4+builtStatus+"\n") 839 for test in dataDict[root][exfile]: 840 if test in self.buildkeys: continue 841 line=indent*2+test 842 fh.write(line+"\n") 843 # Looks nice to have the keys in order 844 #for key in dataDict[root][exfile][test]: 845 for key in "isrun abstracted nsize args requires script".split(): 846 if key not in dataDict[root][exfile][test]: continue 847 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 848 fh.write(line+"\n") 849 fh.write("\n") 850 fh.write("\n") 851 fh.write("\n") 852 return 853 854 def genPetscTests(self,root,dirs,files,dataDict): 855 """ 856 Go through and parse the source files in the directory to generate 857 the examples based on the metadata contained in the source files 858 """ 859 debug=False 860 # Use examplesAnalyze to get what the makefles think are sources 861 #self.examplesAnalyze(root,dirs,files,anlzDict) 862 863 dataDict[root]={} 864 865 for exfile in files: 866 #TST: Until we replace files, still leaving the orginals as is 867 #if not exfile.startswith("new_"+"ex"): continue 868 #if not exfile.startswith("ex"): continue 869 870 # Ignore emacs and other temporary files 871 if exfile.startswith("."): continue 872 if exfile.startswith("#"): continue 873 if exfile.endswith("~"): continue 874 # Only parse source files 875 ext=os.path.splitext(exfile)[-1].lstrip('.') 876 if ext not in LANGS: continue 877 878 # Convenience 879 fullex=os.path.join(root,exfile) 880 if self.verbose: print(' --> '+fullex) 881 dataDict[root].update(testparse.parseTestFile(fullex,0)) 882 if exfile in dataDict[root]: 883 if not self.check_output: 884 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 885 else: 886 self.checkOutput(exfile,root,dataDict[root][exfile]) 887 888 return 889 890 def walktree(self,top): 891 """ 892 Walk a directory tree, starting from 'top' 893 """ 894 if self.check_output: 895 print("Checking for missing output files") 896 self.missing_files=[] 897 898 # Goal of action is to fill this dictionary 899 dataDict={} 900 for root, dirs, files in os.walk(top, topdown=True): 901 dirs.sort() 902 files.sort() 903 if "/tests" not in root and "/tutorials" not in root: continue 904 if "dSYM" in root: continue 905 if os.path.basename(root.rstrip("/")) == 'output': continue 906 if self.verbose: print(root) 907 self.genPetscTests(root,dirs,files,dataDict) 908 909 # If checking output, report results 910 if self.check_output: 911 if self.missing_files: 912 for file in set(self.missing_files): # set uniqifies 913 print(file) 914 sys.exit(1) 915 916 # Now summarize this dictionary 917 if self.verbose: self.genPetscTests_summarize(dataDict) 918 return dataDict 919 920 def gen_gnumake(self, fd): 921 """ 922 Overwrite of the method in the base PETSc class 923 """ 924 def write(stem, srcs): 925 for lang in LANGS: 926 if srcs[lang]['srcs']: 927 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang, srcs=' '.join(srcs[lang]['srcs']))) 928 for pkg in self.pkg_pkgs: 929 srcs = self.gen_pkg(pkg) 930 write('testsrcs-' + pkg, srcs) 931 # Handle dependencies 932 for lang in LANGS: 933 for exfile in srcs[lang]['srcs']: 934 if exfile in srcs[lang]: 935 ex='$(TESTDIR)/'+os.path.splitext(exfile)[0] 936 exfo='$(TESTDIR)/'+os.path.splitext(exfile)[0]+'.o' 937 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 938 if deps: 939 # The executable literally depends on the object file because it is linked 940 fd.write(ex +": " + " ".join(deps) +'\n') 941 # The object file containing 'main' does not normally depend on other object 942 # files, but it does when it includes their modules. This dependency is 943 # overly blunt and could be reduced to only depend on object files for 944 # modules that are used, like "*f90aux.o". 945 fd.write(exfo +": " + " ".join(deps) +'\n') 946 947 return self.gendeps 948 949 def gen_pkg(self, pkg): 950 """ 951 Overwrite of the method in the base PETSc class 952 """ 953 return self.sources[pkg] 954 955 def write_gnumake(self, dataDict, output=None): 956 """ 957 Write out something similar to files from gmakegen.py 958 959 Test depends on script which also depends on source 960 file, but since I don't have a good way generating 961 acting on a single file (oops) just depend on 962 executable which in turn will depend on src file 963 """ 964 # Different options for how to set up the targets 965 compileExecsFirst=False 966 967 # Open file 968 with open(output, 'w') as fd: 969 # Write out the sources 970 gendeps = self.gen_gnumake(fd) 971 972 # Write out the tests and execname targets 973 fd.write("\n#Tests and executables\n") # Delimiter 974 975 for pkg in self.pkg_pkgs: 976 # These grab the ones that are built 977 for lang in LANGS: 978 testdeps=[] 979 for ftest in self.tests[pkg][lang]: 980 test=os.path.basename(ftest) 981 basedir=os.path.dirname(ftest) 982 testdeps.append(nameSpace(test,basedir)) 983 fd.write("test-"+pkg+"."+lang+" := "+' '.join(testdeps)+"\n") 984 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang, pkg, lang)) 985 986 # test targets 987 for ftest in self.tests[pkg][lang]: 988 test=os.path.basename(ftest) 989 basedir=os.path.dirname(ftest) 990 testdir="${TESTDIR}/"+basedir+"/" 991 nmtest=nameSpace(test,basedir) 992 rundir=os.path.join(testdir,test) 993 script=test+".sh" 994 995 # Deps 996 exfile=self.tests[pkg][lang][ftest]['exfile'] 997 fullex=os.path.join(self.srcdir,exfile) 998 localexec=self.tests[pkg][lang][ftest]['exec'] 999 execname=os.path.join(testdir,localexec) 1000 fullscript=os.path.join(testdir,script) 1001 tmpfile=os.path.join(testdir,test,test+".tmp") 1002 1003 # *.counts depends on the script and either executable (will 1004 # be run) or the example source file (SKIP or TODO) 1005 fd.write('%s.counts : %s %s' 1006 % (os.path.join('$(TESTDIR)/counts', nmtest), 1007 fullscript, 1008 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1009 ) 1010 if exfile in self.sources[pkg][lang]: 1011 for dep in self.sources[pkg][lang][exfile]: 1012 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1013 fd.write('\n') 1014 1015 # Now write the args: 1016 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1017 1018 return 1019 1020 def write_db(self, dataDict, testdir): 1021 """ 1022 Write out the dataDict into a pickle file 1023 """ 1024 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1025 pickle.dump(dataDict,fd) 1026 return 1027 1028def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1029 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1030 srcdir=None, testdir=None, check=False): 1031 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1032 testdir=os.path.normpath(testdir) 1033 if petsc_arch: 1034 petsc_arch=petsc_arch.rstrip(os.path.sep) 1035 if len(petsc_arch.split(os.path.sep))>1: 1036 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1037 output = os.path.join(testdir, 'testfiles') 1038 1039 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1040 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1041 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1042 testdir=testdir,check=check) 1043 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1044 if not pEx.check_output: 1045 pEx.write_gnumake(dataDict, output) 1046 pEx.write_db(dataDict, testdir) 1047 1048if __name__ == '__main__': 1049 import optparse 1050 parser = optparse.OptionParser() 1051 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1052 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1053 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1054 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1055 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1056 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1057 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1058 help='Check whether output files are in output director') 1059 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1060 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1061 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1062 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1063 1064 opts, extra_args = parser.parse_args() 1065 if extra_args: 1066 import sys 1067 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1068 exit(1) 1069 if opts.testdir is None: 1070 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1071 1072 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1073 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1074 verbose=opts.verbose, 1075 single_ex=opts.single_executable, srcdir=opts.srcdir, 1076 testdir=opts.testdir, check=opts.check_output) 1077