1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9import shlex 10sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25Case 1. Normal builds: 26 27 +---------------------+----------------------------------+ 28 | PETSC_DIR | <git dir> | 29 +---------------------+----------------------------------+ 30 | PETSC_ARCH | arch-foo | 31 +---------------------+----------------------------------+ 32 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 33 +---------------------+----------------------------------+ 34 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 35 +---------------------+----------------------------------+ 36 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 37 +---------------------+----------------------------------+ 38 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 41 +---------------------+----------------------------------+ 42 43Case 2. From install dir: 44 45 +---------------------+-------------------------------------------------------+ 46 | PETSC_DIR | <prefix dir> | 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_ARCH | '' | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 59 +---------------------+-------------------------------------------------------+ 60 61""" 62 63def install_files(source, destdir): 64 """Install file or directory 'source' to 'destdir'. Does not preserve 65 mode (permissions). 66 """ 67 if not os.path.isdir(destdir): 68 os.makedirs(destdir) 69 if os.path.isdir(source): 70 for name in os.listdir(source): 71 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 72 else: 73 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 74 75def nameSpace(srcfile,srcdir): 76 """ 77 Because the scripts have a non-unique naming, the pretty-printing 78 needs to convey the srcdir and srcfile. There are two ways of doing this. 79 """ 80 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 81 prefix=srcdir.replace("/","_")+"-" 82 nameString=prefix+srcfile 83 return nameString 84 85class generateExamples(Petsc): 86 """ 87 gmakegen.py has basic structure for finding the files, writing out 88 the dependencies, etc. 89 """ 90 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 91 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs) 92 93 self.single_ex=single_ex 94 self.srcdir=srcdir 95 self.check_output=check 96 97 # Set locations to handle movement 98 self.inInstallDir=self.getInInstallDir(thisscriptdir) 99 100 # Special configuration for CI testing 101 if self.petsc_arch.find('valgrind') >= 0: 102 self.conf['PETSCTEST_VALGRIND']=1 103 104 if self.inInstallDir: 105 # Case 2 discussed above 106 # set PETSC_ARCH to install directory to allow script to work in both 107 dirlist=thisscriptdir.split(os.path.sep) 108 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 109 self.arch_dir=installdir 110 if self.srcdir is None: 111 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 112 else: 113 if petsc_arch == '': 114 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 115 # Case 1 discussed above 116 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 117 if self.srcdir is None: 118 self.srcdir=os.path.join(self.petsc_dir,'src') 119 120 self.testroot_dir=os.path.abspath(testdir) 121 122 self.verbose=verbose 123 # Whether to write out a useful debugging 124 self.summarize=True if verbose else False 125 126 # For help in setting the requirements 127 self.precision_types="__fp16 single double __float128".split() 128 self.integer_types="int32 int64 long32 long64".split() 129 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 130 131 # Things that are not test 132 self.buildkeys=testparse.buildkeys 133 134 # Adding a dictionary for storing sources, objects, and tests 135 # to make building the dependency tree easier 136 self.sources={} 137 self.objects={} 138 self.tests={} 139 for pkg in self.pkg_pkgs: 140 self.sources[pkg]={} 141 self.objects[pkg]=[] 142 self.tests[pkg]={} 143 for lang in LANGS: 144 self.sources[pkg][lang]={} 145 self.sources[pkg][lang]['srcs']=[] 146 self.tests[pkg][lang]={} 147 148 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 149 150 self.indent=" " 151 if self.verbose: print('Finishing the constructor') 152 return 153 154 def srcrelpath(self,rdir): 155 """ 156 Get relative path to source directory 157 """ 158 return os.path.relpath(rdir,self.srcdir) 159 160 def getInInstallDir(self,thisscriptdir): 161 """ 162 When PETSc is installed then this file in installed in: 163 <PREFIX>/share/petsc/examples/config/gmakegentest.py 164 otherwise the path is: 165 <PETSC_DIR>/config/gmakegentest.py 166 We use this difference to determine if we are in installdir 167 """ 168 dirlist=thisscriptdir.split(os.path.sep) 169 if len(dirlist)>4: 170 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 171 if lastfour==os.path.join('share','petsc','examples','config'): 172 return True 173 else: 174 return False 175 else: 176 return False 177 178 def getLanguage(self,srcfile): 179 """ 180 Based on the source, determine associated language as found in gmakegen.LANGS 181 Can we just return srcext[1:] now? 182 """ 183 langReq=None 184 srcext = getlangext(srcfile) 185 if srcext in ".F90".split(): langReq="F90" 186 if srcext in ".F".split(): langReq="F" 187 if srcext in ".cxx".split(): langReq="cxx" 188 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 189 if srcext in ".hip.cxx".split(): langReq="hip_cxx" 190 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getAltList(self,output_file,srcdir): 198 ''' Calculate AltList based on output file-- see 199 src/snes/tutorials/output/ex22*.out 200 ''' 201 altlist=[output_file] 202 basefile = getlangsplit(output_file) 203 for i in range(1,9): 204 altroot=basefile+"_alt" 205 if i > 1: altroot=altroot+"_"+str(i) 206 af=altroot+".out" 207 srcaf=os.path.join(srcdir,af) 208 fullaf=os.path.join(self.petsc_dir,srcaf) 209 if os.path.isfile(fullaf): altlist.append(srcaf) 210 211 return altlist 212 213 def _getLoopVars(self,inDict,testname, isSubtest=False): 214 """ 215 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 216 Return: 217 inDict['args']: -ksp_monitor 218 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 219 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 220 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 221 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 222 subst should be passed in instead of inDict 223 """ 224 loopVars={}; newargs=[] 225 lsuffix='+' 226 argregex = re.compile(' (?=-[a-zA-Z])') 227 from testparse import parseLoopArgs 228 for key in inDict: 229 if key in ('SKIP', 'regexes'): 230 continue 231 akey=('subargs' if key=='args' else key) # what to assign 232 if akey not in inDict: inDict[akey]='' 233 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 234 # Always generate a loop over nsize, even if there is only one value 235 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 236 keystr = str(inDict[key]) 237 varlist = [] 238 for varset in argregex.split(keystr): 239 if not varset.strip(): continue 240 if '{{' in varset: 241 keyvar,lvars,ftype=parseLoopArgs(varset) 242 if akey not in loopVars: loopVars[akey]={} 243 varlist.append(keyvar) 244 loopVars[akey][keyvar]=[keyvar,lvars] 245 if akey=='nsize': 246 if len(lvars.split()) > 1: 247 lsuffix += akey +'-${i' + keyvar + '}' 248 else: 249 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 250 lsuffix+=keyvar+'-${i' + keyvar + '}_' 251 else: 252 if key=='args': 253 newargs.append(varset.strip()) 254 if varlist: 255 loopVars[akey]['varlist']=varlist 256 257 # For subtests, args are always substituted in (not top level) 258 if isSubtest: 259 inDict['subargs'] += " "+" ".join(newargs) 260 inDict['args']='' 261 if 'label_suffix' in inDict: 262 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 263 else: 264 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 265 else: 266 if loopVars: 267 inDict['args'] = ' '.join(newargs) 268 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 269 return loopVars 270 271 def getArgLabel(self,testDict): 272 """ 273 In all of the arguments in the test dictionary, create a simple 274 string for searching within the makefile system. For simplicity in 275 search, remove "-", for strings, etc. 276 Also, concatenate the arg commands 277 For now, ignore nsize -- seems hard to search for anyway 278 """ 279 # Collect all of the args associated with a test 280 argStr=("" if 'args' not in testDict else testDict['args']) 281 if 'subtests' in testDict: 282 for stest in testDict["subtests"]: 283 sd=testDict[stest] 284 argStr=argStr+("" if 'args' not in sd else sd['args']) 285 286 # Now go through and cleanup 287 argStr=re.sub('{{(.*?)}}',"",argStr) 288 argStr=re.sub('-'," ",argStr) 289 for digit in string.digits: argStr=re.sub(digit," ",argStr) 290 argStr=re.sub(r"\.","",argStr) 291 argStr=re.sub(",","",argStr) 292 argStr=re.sub(r'\+',' ',argStr) 293 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 294 return argStr.strip() 295 296 def addToSources(self,exfile,rpath,srcDict): 297 """ 298 Put into data structure that allows easy generation of makefile 299 """ 300 pkg=rpath.split(os.path.sep)[0] 301 relpfile=os.path.join(rpath,exfile) 302 lang=self.getLanguage(exfile) 303 if not lang: return 304 if pkg not in self.sources: return 305 self.sources[pkg][lang]['srcs'].append(relpfile) 306 self.sources[pkg][lang][relpfile] = [] 307 if 'depends' in srcDict: 308 depSrcList=srcDict['depends'].split() 309 for depSrc in depSrcList: 310 depObj = getlangsplit(depSrc)+'.o' 311 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 312 313 # In gmakefile, ${TESTDIR} var specifies the object compilation 314 testsdir=rpath+"/" 315 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 316 self.objects[pkg].append(objfile) 317 return 318 319 def addToTests(self,test,rpath,exfile,execname,testDict): 320 """ 321 Put into data structure that allows easy generation of makefile 322 Organized by languages to allow testing of languages 323 """ 324 pkg=rpath.split("/")[0] 325 nmtest=os.path.join(rpath,test) 326 lang=self.getLanguage(exfile) 327 if not lang: return 328 if pkg not in self.tests: return 329 self.tests[pkg][lang][nmtest]={} 330 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 331 self.tests[pkg][lang][nmtest]['exec']=execname 332 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 333 return 334 335 def getExecname(self,exfile,rpath): 336 """ 337 Generate bash script using template found next to this file. 338 This file is read in at constructor time to avoid file I/O 339 """ 340 if self.single_ex: 341 execname=rpath.split("/")[1]+"-ex" 342 else: 343 execname=getlangsplit(exfile) 344 return execname 345 346 def getSubstVars(self,testDict,rpath,testname): 347 """ 348 Create a dictionary with all of the variables that get substituted 349 into the template commands found in example_template.py 350 """ 351 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 352 if 'nsize' not in testDict: testDict['nsize'] = '1' 353 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 354 subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'} 355 356 # Now do other variables 357 subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env'])) 358 subst['execname']=testDict['execname'] 359 subst['error']='' 360 if 'filter' in testDict: 361 if testDict['filter'].startswith("Error:"): 362 subst['error']="Error" 363 subst['filter']=testDict['filter'].lstrip("Error:") 364 else: 365 subst['filter']=testDict['filter'] 366 367 # Others 368 subst['subargs']='' # Default. For variables override 369 subst['srcdir']=os.path.join(self.srcdir, rpath) 370 subst['label_suffix']='' 371 subst['comments']="\n#".join(subst['comments'].split("\n")) 372 if subst['comments']: subst['comments']="#"+subst['comments'] 373 subst['executable']="../"+subst['execname'] 374 subst['testroot']=self.testroot_dir 375 subst['testname']=testname 376 dp = self.conf.get('DATAFILESPATH','') 377 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 378 379 # This is used to label some matrices 380 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 381 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 382 383 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 384 385 #Conf vars 386 if self.petsc_arch.find('valgrind')>=0: 387 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 388 else: 389 subst['mpiexec']=self.conf['MPIEXEC'] 390 subst['mpiexec_tail']=self.conf['MPIEXEC_TAIL'] 391 subst['pkg_name']=self.pkg_name 392 subst['pkg_dir']=self.pkg_dir 393 subst['pkg_arch']=self.pkg_arch 394 subst['CONFIG_DIR']=thisscriptdir 395 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 396 subst['diff']=self.conf['DIFF'] 397 subst['rm']=self.conf['RM'] 398 subst['grep']=self.conf['GREP'] 399 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 400 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 401 402 # Output file is special because of subtests override 403 defroot = testparse.getDefaultOutputFileRoot(testname) 404 if 'output_file' not in testDict: 405 subst['output_file']="output/"+defroot+".out" 406 subst['redirect_file']=defroot+".tmp" 407 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 408 409 # Add in the full path here. 410 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 411 412 subst['regexes']={} 413 for subkey in subst: 414 if subkey=='regexes': continue 415 if not isinstance(subst[subkey],str): continue 416 patt="@"+subkey.upper()+"@" 417 subst['regexes'][subkey]=re.compile(patt) 418 419 return subst 420 421 def _substVars(self,subst,origStr): 422 """ 423 Substitute variables 424 """ 425 Str=origStr 426 for subkey, subvalue in subst.items(): 427 if subkey=='regexes': continue 428 if not isinstance(subvalue,str): continue 429 if subkey.upper() not in Str: continue 430 Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str) 431 return Str 432 433 def getCmds(self,subst,i, debug=False): 434 """ 435 Generate bash script using template found next to this file. 436 This file is read in at constructor time to avoid file I/O 437 """ 438 nindnt=i # the start and has to be consistent with below 439 cmdindnt=self.indent*nindnt 440 cmdLines="" 441 442 # MPI is the default -- but we have a few odd commands 443 if subst['temporaries']: 444 if '*' in subst['temporaries']: 445 raise RuntimeError('{}/{}: list of temporary files to remove may not include wildcards'.format(subst['srcdir'], subst['execname'])) 446 cmd=cmdindnt+self._substVars(subst,example_template.preclean) 447 cmdLines+=cmd+"\n" 448 if not subst['command']: 449 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 450 else: 451 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 452 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 453 454 cmdLines+=cmdindnt+'if test $res = 0; then\n' 455 diffindnt=self.indent*(nindnt+1) 456 457 # Do some checks on existence of output_file and alt files 458 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 459 if not subst['TODO']: 460 print("Warning: "+subst['output_file']+" not found.") 461 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 462 463 # altlist always has output_file 464 if len(altlist)==1: 465 cmd=diffindnt+self._substVars(subst,example_template.difftest) 466 else: 467 if debug: print("Found alt files: ",altlist) 468 # Have to do it by hand a bit because of variable number of alt files 469 rf=subst['redirect_file'] 470 cmd=diffindnt+example_template.difftest.split('@')[0] 471 for i in range(len(altlist)): 472 af=altlist[i] 473 cmd+=af+' '+rf 474 if i!=len(altlist)-1: 475 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 476 cmd+=' || ${diff_exe} ' 477 else: 478 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 479 cmd+=subst['label_suffix']+' ""' # Quotes are painful 480 cmdLines+=cmd+"\n" 481 cmdLines+=cmdindnt+'else\n' 482 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 483 cmdLines+=cmdindnt+'fi\n' 484 return cmdLines 485 486 def _writeTodoSkip(self,fh,tors,reasons,footer): 487 """ 488 Write out the TODO and SKIP lines in the file 489 The TODO or SKIP variable, tors, should be lower case 490 """ 491 TORS=tors.upper() 492 template=eval("example_template."+tors+"line") 493 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 494 tab = '' 495 if reasons: 496 fh.write('if ! $force; then\n') 497 tab = tab + ' ' 498 if reasons == ["Requires DATAFILESPATH"]: 499 # The only reason not to run is DATAFILESPATH, which we check at run-time 500 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 501 tab = tab + ' ' 502 if reasons: 503 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 504 fh.write(tab+footer+"\n") 505 fh.write(tab+"exit\n") 506 if reasons == ["Requires DATAFILESPATH"]: 507 fh.write(' fi\n') 508 if reasons: 509 fh.write('fi\n') 510 fh.write('\n\n') 511 return 512 513 def getLoopVarsHead(self,loopVars,i,usedVars={}): 514 """ 515 Generate a nicely indented string with the format loops 516 Here is what the data structure looks like 517 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 518 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 519 loopVars['subargs']['pc_type']=["j","cholesky sor"] 520 """ 521 outstr=''; indnt=self.indent 522 523 for key in loopVars: 524 for var in loopVars[key]['varlist']: 525 varval=loopVars[key][var] 526 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 527 outstr += "\n\n" 528 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 varval=loopVars[key][var] 532 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 533 i = i + 1 534 return (outstr,i) 535 536 def getLoopVarsFoot(self,loopVars,i): 537 outstr=''; indnt=self.indent 538 for key in loopVars: 539 for var in loopVars[key]['varlist']: 540 i = i - 1 541 outstr += indnt * i + "done\n" 542 return (outstr,i) 543 544 def genRunScript(self,testname,root,isRun,srcDict): 545 """ 546 Generate bash script using template found next to this file. 547 This file is read in at constructor time to avoid file I/O 548 """ 549 def opener(path,flags,*args,**kwargs): 550 kwargs.setdefault('mode',0o755) 551 return os.open(path,flags,*args,**kwargs) 552 553 # runscript_dir directory has to be consistent with gmakefile 554 testDict=srcDict[testname] 555 rpath=self.srcrelpath(root) 556 runscript_dir=os.path.join(self.testroot_dir,rpath) 557 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 558 with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh: 559 560 # Get variables to go into shell scripts. last time testDict used 561 subst=self.getSubstVars(testDict,rpath,testname) 562 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 563 if 'subtests' in testDict: 564 # The subtests inherit inDict, so we don't need top-level loops. 565 loopVars = {} 566 567 #Handle runfiles 568 for lfile in subst.get('localrunfiles','').split(): 569 install_files(os.path.join(root, lfile), 570 os.path.join(runscript_dir, os.path.dirname(lfile))) 571 # Check subtests for local runfiles 572 for stest in subst.get("subtests",[]): 573 for lfile in testDict[stest].get('localrunfiles','').split(): 574 install_files(os.path.join(root, lfile), 575 os.path.join(runscript_dir, os.path.dirname(lfile))) 576 577 # Now substitute the key variables into the header and footer 578 header=self._substVars(subst,example_template.header) 579 # The header is done twice to enable @...@ in header 580 header=self._substVars(subst,header) 581 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 582 583 # Start writing the file 584 fh.write(header+"\n") 585 586 # If there is a TODO or a SKIP then we do it before writing out the 587 # rest of the command (which is useful for working on the test) 588 # SKIP and TODO can be for the source file or for the runs 589 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 590 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 591 592 j=0 # for indentation 593 594 if loopVars: 595 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 596 if (loopHead): fh.write(loopHead+"\n") 597 598 # Subtests are special 599 allLoopVars=list(loopVars.keys()) 600 if 'subtests' in testDict: 601 substP=subst # Subtests can inherit args but be careful 602 k=0 # for label suffixes 603 for stest in testDict["subtests"]: 604 subst=substP.copy() 605 subst.update(testDict[stest]) 606 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 607 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 608 if sLoopVars: 609 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 610 allLoopVars+=list(sLoopVars.keys()) 611 fh.write(sLoopHead+"\n") 612 fh.write(self.getCmds(subst,j)+"\n") 613 if sLoopVars: 614 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 615 fh.write(sLoopFoot+"\n") 616 else: 617 fh.write(self.getCmds(subst,j)+"\n") 618 619 if loopVars: 620 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 621 fh.write(loopFoot+"\n") 622 623 fh.write(footer+"\n") 624 return 625 626 def genScriptsAndInfo(self,exfile,root,srcDict): 627 """ 628 Generate scripts from the source file, determine if built, etc. 629 For every test in the exfile with info in the srcDict: 630 1. Determine if it needs to be run for this arch 631 2. Generate the script 632 3. Generate the data needed to write out the makefile in a 633 convenient way 634 All tests are *always* run, but some may be SKIP'd per the TAP standard 635 """ 636 debug=False 637 rpath=self.srcrelpath(root) 638 execname=self.getExecname(exfile,rpath) 639 isBuilt=self._isBuilt(exfile,srcDict) 640 for test in srcDict.copy(): 641 if test in self.buildkeys: continue 642 isRun=self._isRun(srcDict[test]) 643 # if the next two lines are dropped all scripts are generating included the unneeded 644 # if the unneeded are generated when run they will skip their tests automatically 645 # not generating them saves setup time 646 allow = False 647 if 'SKIP' in srcDict[test]: 648 allow = srcDict[test]['SKIP'] in [['Requires DATAFILESPATH'], ['PETSC_HAVE_PYVISTA requirement not met']] 649 if not isRun and not allow: 650 del srcDict[test] 651 continue 652 if 'TODO' in srcDict[test]: 653 del srcDict[test] 654 continue 655 srcDict[test]['execname']=execname # Convenience in generating scripts 656 self.genRunScript(test,root,isRun,srcDict) 657 srcDict[test]['isrun']=isRun 658 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 659 660 # This adds to datastructure for building deps 661 if isBuilt: self.addToSources(exfile,rpath,srcDict) 662 return 663 664 def _isBuilt(self,exfile,srcDict): 665 """ 666 Determine if this file should be built. 667 """ 668 # Get the language based on file extension 669 srcDict['SKIP'] = [] 670 lang=self.getLanguage(exfile) 671 if (lang=="F" or lang=="F90"): 672 if not self.have_fortran: 673 srcDict["SKIP"].append("Fortran required for this test") 674 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 675 srcDict["SKIP"].append("CUDA required for this test") 676 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 677 srcDict["SKIP"].append("HIP required for this test") 678 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 679 srcDict["SKIP"].append("SYCL required for this test") 680 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 681 srcDict["SKIP"].append("KOKKOS required for this test") 682 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 683 srcDict["SKIP"].append("RAJA required for this test") 684 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 685 srcDict["SKIP"].append("C++ required for this test") 686 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 687 srcDict["SKIP"].append("C++ required for this test") 688 689 # Deprecated source files 690 if srcDict.get("TODO"): 691 return False 692 693 # isRun can work with srcDict to handle the requires 694 if "requires" in srcDict: 695 if srcDict["requires"]: 696 return self._isRun(srcDict) 697 698 return srcDict['SKIP'] == [] 699 700 def _isRun(self,testDict, debug=False): 701 """ 702 Based on the requirements listed in the src file and the petscconf.h 703 info, determine whether this test should be run or not. 704 """ 705 indent=" " 706 707 if 'SKIP' not in testDict: 708 testDict['SKIP'] = [] 709 # MPI requirements 710 if 'MPI_IS_MPIUNI' in self.conf: 711 if testDict.get('nsize', '1') != '1': 712 testDict['SKIP'].append("Parallel test with serial build") 713 714 # The requirements for the test are the sum of all the run subtests 715 if 'subtests' in testDict: 716 if 'requires' not in testDict: testDict['requires']="" 717 for stest in testDict['subtests']: 718 if 'requires' in testDict[stest]: 719 testDict['requires']+=" "+testDict[stest]['requires'] 720 if testDict[stest].get('nsize', '1') != '1': 721 testDict['SKIP'].append("Parallel test with serial build") 722 break 723 724 # Now go through all requirements 725 if 'requires' in testDict: 726 for requirement in testDict['requires'].split(): 727 requirement=requirement.strip() 728 if not requirement: continue 729 if debug: print(indent+"Requirement: ", requirement) 730 isNull=False 731 if requirement.startswith("!"): 732 requirement=requirement[1:]; isNull=True 733 # 32-bit vs 64-bit pointers 734 if requirement == "64bitptr": 735 if self.conf['PETSC_SIZEOF_VOID_P']==8: 736 if isNull: 737 testDict['SKIP'].append("not 64bit-ptr required") 738 continue 739 continue # Success 740 elif not isNull: 741 testDict['SKIP'].append("64bit-ptr required") 742 continue 743 # Precision requirement for reals 744 if requirement in self.precision_types: 745 if self.conf['PETSC_PRECISION']==requirement: 746 if isNull: 747 testDict['SKIP'].append("not "+requirement+" required") 748 continue 749 continue # Success 750 elif not isNull: 751 testDict['SKIP'].append(requirement+" required") 752 continue 753 # Precision requirement for ints 754 if requirement in self.integer_types: 755 if requirement=="int32": 756 if self.conf['PETSC_SIZEOF_INT']==4: 757 if isNull: 758 testDict['SKIP'].append("not int32 required") 759 continue 760 continue # Success 761 elif not isNull: 762 testDict['SKIP'].append("int32 required") 763 continue 764 if requirement=="int64": 765 if self.conf['PETSC_SIZEOF_INT']==8: 766 if isNull: 767 testDict['SKIP'].append("NOT int64 required") 768 continue 769 continue # Success 770 elif not isNull: 771 testDict['SKIP'].append("int64 required") 772 continue 773 if requirement.startswith("long"): 774 reqsize = int(requirement[4:])//8 775 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 776 if longsize==reqsize: 777 if isNull: 778 testDict['SKIP'].append("not %s required" % requirement) 779 continue 780 continue # Success 781 elif not isNull: 782 testDict['SKIP'].append("%s required" % requirement) 783 continue 784 # Datafilespath 785 if requirement=="datafilespath" and not isNull: 786 testDict['SKIP'].append("Requires DATAFILESPATH") 787 continue 788 # Defines -- not sure I have comments matching 789 if "defined(" in requirement.lower(): 790 reqdef=requirement.split("(")[1].split(")")[0] 791 if reqdef in self.conf: 792 if isNull: 793 testDict['SKIP'].append("Null requirement not met: "+requirement) 794 continue 795 continue # Success 796 elif not isNull: 797 testDict['SKIP'].append("Required: "+requirement) 798 continue 799 800 # Rest should be packages that we can just get from conf 801 if requirement in ["complex","debug"]: 802 petscconfvar="PETSC_USE_"+requirement.upper() 803 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 804 else: 805 petscconfvar="PETSC_HAVE_"+requirement.upper() 806 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 807 petsccv = self.conf.get(petscconfvar) 808 pkgcv = self.conf.get(pkgconfvar) 809 810 if petsccv or pkgcv: 811 if isNull: 812 if petsccv: 813 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 814 continue 815 else: 816 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 817 continue 818 continue # Success 819 elif not isNull: 820 if not petsccv and not pkgcv: 821 if debug: print("requirement not found: ", requirement) 822 if self.pkg_name == 'petsc': 823 testDict['SKIP'].append(petscconfvar+" requirement not met") 824 else: 825 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 826 continue 827 return testDict['SKIP'] == [] 828 829 def checkOutput(self,exfile,root,srcDict): 830 """ 831 Check and make sure the output files are in the output directory 832 """ 833 debug=False 834 rpath=self.srcrelpath(root) 835 for test in srcDict: 836 if test in self.buildkeys: continue 837 if debug: print(rpath, exfile, test) 838 if 'output_file' in srcDict[test]: 839 output_file=srcDict[test]['output_file'] 840 else: 841 defroot = testparse.getDefaultOutputFileRoot(test) 842 if 'TODO' in srcDict[test]: continue 843 output_file="output/"+defroot+".out" 844 845 fullout=os.path.join(root,output_file) 846 if debug: print("---> ",fullout) 847 if not os.path.exists(fullout): 848 self.missing_files.append(fullout) 849 850 return 851 852 def genPetscTests_summarize(self,dataDict): 853 """ 854 Required method to state what happened 855 """ 856 if not self.summarize: return 857 indent=" " 858 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 859 with open(fhname, "w") as fh: 860 for root in dataDict: 861 relroot=self.srcrelpath(root) 862 pkg=relroot.split("/")[1] 863 if not pkg in self.sources: continue 864 fh.write(relroot+"\n") 865 allSrcs=[] 866 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 867 for exfile in dataDict[root]: 868 # Basic information 869 rfile=os.path.join(relroot,exfile) 870 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 871 fh.write(indent+exfile+indent*4+builtStatus+"\n") 872 for test in dataDict[root][exfile]: 873 if test in self.buildkeys: continue 874 line=indent*2+test 875 fh.write(line+"\n") 876 # Looks nice to have the keys in order 877 #for key in dataDict[root][exfile][test]: 878 for key in "isrun abstracted nsize args requires script".split(): 879 if key not in dataDict[root][exfile][test]: continue 880 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 881 fh.write(line+"\n") 882 fh.write("\n") 883 fh.write("\n") 884 fh.write("\n") 885 return 886 887 def genPetscTests(self,root,dirs,files,dataDict): 888 """ 889 Go through and parse the source files in the directory to generate 890 the examples based on the metadata contained in the source files 891 """ 892 debug=False 893 894 data = {} 895 for exfile in files: 896 #TST: Until we replace files, still leaving the originals as is 897 #if not exfile.startswith("new_"+"ex"): continue 898 #if not exfile.startswith("ex"): continue 899 900 # Ignore emacs and other temporary files 901 if exfile.startswith((".", "#")) or exfile.endswith("~"): continue 902 # Only parse source files 903 ext=getlangext(exfile).lstrip('.').replace('.','_') 904 if ext not in LANGS: continue 905 906 # Convenience 907 fullex=os.path.join(root,exfile) 908 if self.verbose: print(' --> '+fullex) 909 data.update(testparse.parseTestFile(fullex,0)) 910 if exfile in data: 911 if self.check_output: 912 self.checkOutput(exfile,root,data[exfile]) 913 else: 914 self.genScriptsAndInfo(exfile,root,data[exfile]) 915 916 dataDict[root] = data 917 return 918 919 def walktree(self,top): 920 """ 921 Walk a directory tree, starting from 'top' 922 """ 923 if self.check_output: 924 print("Checking for missing output files") 925 self.missing_files=[] 926 927 # Goal of action is to fill this dictionary 928 dataDict={} 929 for root, dirs, files in os.walk(top, topdown=True): 930 dirs.sort() 931 files.sort() 932 if "/tests" not in root and "/tutorials" not in root: continue 933 if "dSYM" in root: continue 934 if "tutorials"+os.sep+"build" in root: continue 935 if os.path.basename(root.rstrip("/")) == 'output': continue 936 if self.verbose: print(root) 937 self.genPetscTests(root,dirs,files,dataDict) 938 939 # If checking output, report results 940 if self.check_output: 941 if self.missing_files: 942 for file in set(self.missing_files): # set uniqifies 943 print(file) 944 sys.exit(1) 945 946 # Now summarize this dictionary 947 if self.verbose: self.genPetscTests_summarize(dataDict) 948 return dataDict 949 950 def gen_gnumake(self, fd): 951 """ 952 Overwrite of the method in the base PETSc class 953 """ 954 def write(stem, srcs): 955 for lang in LANGS: 956 if srcs[lang]['srcs']: 957 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 958 for pkg in self.pkg_pkgs: 959 srcs = self.gen_pkg(pkg) 960 write('testsrcs-' + pkg, srcs) 961 # Handle dependencies 962 for lang in LANGS: 963 for exfile in srcs[lang]['srcs']: 964 if exfile in srcs[lang]: 965 ex='$(TESTDIR)/'+getlangsplit(exfile) 966 exfo=ex+'.o' 967 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 968 if deps: 969 # The executable literally depends on the object file because it is linked 970 fd.write(ex +": " + " ".join(deps) +'\n') 971 # The object file containing 'main' does not normally depend on other object 972 # files, but it does when it includes their modules. This dependency is 973 # overly blunt and could be reduced to only depend on object files for 974 # modules that are used, like "*f90aux.o". 975 fd.write(exfo +": " + " ".join(deps) +'\n') 976 977 return self.gendeps 978 979 def gen_pkg(self, pkg): 980 """ 981 Overwrite of the method in the base PETSc class 982 """ 983 return self.sources[pkg] 984 985 def write_gnumake(self, dataDict, output=None): 986 """ 987 Write out something similar to files from gmakegen.py 988 989 Test depends on script which also depends on source 990 file, but since I don't have a good way generating 991 acting on a single file (oops) just depend on 992 executable which in turn will depend on src file 993 """ 994 # Different options for how to set up the targets 995 compileExecsFirst=False 996 997 # Open file 998 with open(output, 'w') as fd: 999 # Write out the sources 1000 gendeps = self.gen_gnumake(fd) 1001 1002 # Write out the tests and execname targets 1003 fd.write("\n#Tests and executables\n") # Delimiter 1004 1005 for pkg in self.pkg_pkgs: 1006 # These grab the ones that are built 1007 for lang in LANGS: 1008 testdeps=[] 1009 for ftest in self.tests[pkg][lang]: 1010 test=os.path.basename(ftest) 1011 basedir=os.path.dirname(ftest) 1012 testdeps.append(nameSpace(test,basedir)) 1013 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1014 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1015 1016 # test targets 1017 for ftest in self.tests[pkg][lang]: 1018 test=os.path.basename(ftest) 1019 basedir=os.path.dirname(ftest) 1020 testdir="${TESTDIR}/"+basedir+"/" 1021 nmtest=nameSpace(test,basedir) 1022 rundir=os.path.join(testdir,test) 1023 script=test+".sh" 1024 1025 # Deps 1026 exfile=self.tests[pkg][lang][ftest]['exfile'] 1027 fullex=os.path.join(self.srcdir,exfile) 1028 localexec=self.tests[pkg][lang][ftest]['exec'] 1029 execname=os.path.join(testdir,localexec) 1030 fullscript=os.path.join(testdir,script) 1031 tmpfile=os.path.join(testdir,test,test+".tmp") 1032 1033 # *.counts depends on the script and either executable (will 1034 # be run) or the example source file (SKIP or TODO) 1035 fd.write('%s.counts : %s %s' 1036 % (os.path.join('$(TESTDIR)/counts', nmtest), 1037 fullscript, 1038 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1039 ) 1040 if exfile in self.sources[pkg][lang]: 1041 for dep in self.sources[pkg][lang][exfile]: 1042 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1043 fd.write('\n') 1044 1045 # Now write the args: 1046 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1047 1048 return 1049 1050 def write_db(self, dataDict, testdir): 1051 """ 1052 Write out the dataDict into a pickle file 1053 """ 1054 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1055 pickle.dump(dataDict,fd) 1056 return 1057 1058def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1059 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1060 srcdir=None, testdir=None, check=False): 1061 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1062 testdir=os.path.normpath(testdir) 1063 if petsc_arch: 1064 petsc_arch=petsc_arch.rstrip(os.path.sep) 1065 if len(petsc_arch.split(os.path.sep))>1: 1066 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1067 output = os.path.join(testdir, 'testfiles') 1068 1069 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1070 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1071 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1072 testdir=testdir,check=check) 1073 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1074 if not pEx.check_output: 1075 pEx.write_gnumake(dataDict, output) 1076 pEx.write_db(dataDict, testdir) 1077 1078if __name__ == '__main__': 1079 import optparse 1080 parser = optparse.OptionParser() 1081 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1082 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1083 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1084 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1085 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1086 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1087 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1088 help='Check whether output files are in output directory') 1089 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1090 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1091 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1092 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1093 1094 opts, extra_args = parser.parse_args() 1095 if extra_args: 1096 import sys 1097 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1098 exit(1) 1099 if opts.testdir is None: 1100 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1101 1102 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1103 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1104 verbose=opts.verbose, 1105 single_ex=opts.single_executable, srcdir=opts.srcdir, 1106 testdir=opts.testdir, check=opts.check_output) 1107