1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9import shlex 10sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78def nameSpace(srcfile,srcdir): 79 """ 80 Because the scripts have a non-unique naming, the pretty-printing 81 needs to convey the srcdir and srcfile. There are two ways of doing this. 82 """ 83 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 84 prefix=srcdir.replace("/","_")+"-" 85 nameString=prefix+srcfile 86 return nameString 87 88class generateExamples(Petsc): 89 """ 90 gmakegen.py has basic structure for finding the files, writing out 91 the dependencies, etc. 92 """ 93 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 94 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs) 95 96 self.single_ex=single_ex 97 self.srcdir=srcdir 98 self.check_output=check 99 100 # Set locations to handle movement 101 self.inInstallDir=self.getInInstallDir(thisscriptdir) 102 103 # Special configuration for CI testing 104 if self.petsc_arch.find('valgrind') >= 0: 105 self.conf['PETSCTEST_VALGRIND']=1 106 107 if self.inInstallDir: 108 # Case 2 discussed above 109 # set PETSC_ARCH to install directory to allow script to work in both 110 dirlist=thisscriptdir.split(os.path.sep) 111 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 112 self.arch_dir=installdir 113 if self.srcdir is None: 114 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 115 else: 116 if petsc_arch == '': 117 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 118 # Case 1 discussed above 119 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 120 if self.srcdir is None: 121 self.srcdir=os.path.join(self.petsc_dir,'src') 122 123 self.testroot_dir=os.path.abspath(testdir) 124 125 self.verbose=verbose 126 # Whether to write out a useful debugging 127 self.summarize=True if verbose else False 128 129 # For help in setting the requirements 130 self.precision_types="__fp16 single double __float128".split() 131 self.integer_types="int32 int64 long32 long64".split() 132 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 133 134 # Things that are not test 135 self.buildkeys=testparse.buildkeys 136 137 # Adding a dictionary for storing sources, objects, and tests 138 # to make building the dependency tree easier 139 self.sources={} 140 self.objects={} 141 self.tests={} 142 for pkg in self.pkg_pkgs: 143 self.sources[pkg]={} 144 self.objects[pkg]=[] 145 self.tests[pkg]={} 146 for lang in LANGS: 147 self.sources[pkg][lang]={} 148 self.sources[pkg][lang]['srcs']=[] 149 self.tests[pkg][lang]={} 150 151 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 152 153 self.indent=" " 154 if self.verbose: print('Finishing the constructor') 155 return 156 157 def srcrelpath(self,rdir): 158 """ 159 Get relative path to source directory 160 """ 161 return os.path.relpath(rdir,self.srcdir) 162 163 def getInInstallDir(self,thisscriptdir): 164 """ 165 When PETSc is installed then this file in installed in: 166 <PREFIX>/share/petsc/examples/config/gmakegentest.py 167 otherwise the path is: 168 <PETSC_DIR>/config/gmakegentest.py 169 We use this difference to determine if we are in installdir 170 """ 171 dirlist=thisscriptdir.split(os.path.sep) 172 if len(dirlist)>4: 173 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 174 if lastfour==os.path.join('share','petsc','examples','config'): 175 return True 176 else: 177 return False 178 else: 179 return False 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:] now? 185 """ 186 langReq=None 187 srcext = getlangext(srcfile) 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 192 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 193 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 194 if srcext in ".cpp".split(): langReq="cpp" 195 if srcext == ".cu": langReq="cu" 196 if srcext == ".c": langReq="c" 197 #if not langReq: print("ERROR: ", srcext, srcfile) 198 return langReq 199 200 def _getAltList(self,output_file,srcdir): 201 ''' Calculate AltList based on output file-- see 202 src/snes/tutorials/output/ex22*.out 203 ''' 204 altlist=[output_file] 205 basefile = getlangsplit(output_file) 206 for i in range(1,9): 207 altroot=basefile+"_alt" 208 if i > 1: altroot=altroot+"_"+str(i) 209 af=altroot+".out" 210 srcaf=os.path.join(srcdir,af) 211 fullaf=os.path.join(self.petsc_dir,srcaf) 212 if os.path.isfile(fullaf): altlist.append(srcaf) 213 214 return altlist 215 216 217 def _getLoopVars(self,inDict,testname, isSubtest=False): 218 """ 219 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 220 Return: 221 inDict['args']: -ksp_monitor 222 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 223 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 224 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 225 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 226 subst should be passed in instead of inDict 227 """ 228 loopVars={}; newargs=[] 229 lsuffix='+' 230 argregex = re.compile(' (?=-[a-zA-Z])') 231 from testparse import parseLoopArgs 232 for key in inDict: 233 if key in ('SKIP', 'regexes'): 234 continue 235 akey=('subargs' if key=='args' else key) # what to assign 236 if akey not in inDict: inDict[akey]='' 237 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 238 # Always generate a loop over nsize, even if there is only one value 239 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 240 keystr = str(inDict[key]) 241 varlist = [] 242 for varset in argregex.split(keystr): 243 if not varset.strip(): continue 244 if '{{' in varset: 245 keyvar,lvars,ftype=parseLoopArgs(varset) 246 if akey not in loopVars: loopVars[akey]={} 247 varlist.append(keyvar) 248 loopVars[akey][keyvar]=[keyvar,lvars] 249 if akey=='nsize': 250 if len(lvars.split()) > 1: 251 lsuffix += akey +'-${i' + keyvar + '}' 252 else: 253 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 254 lsuffix+=keyvar+'-${i' + keyvar + '}_' 255 else: 256 if key=='args': 257 newargs.append(varset.strip()) 258 if varlist: 259 loopVars[akey]['varlist']=varlist 260 261 # For subtests, args are always substituted in (not top level) 262 if isSubtest: 263 inDict['subargs'] += " "+" ".join(newargs) 264 inDict['args']='' 265 if 'label_suffix' in inDict: 266 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 267 else: 268 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 269 else: 270 if loopVars: 271 inDict['args'] = ' '.join(newargs) 272 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 273 return loopVars 274 275 def getArgLabel(self,testDict): 276 """ 277 In all of the arguments in the test dictionary, create a simple 278 string for searching within the makefile system. For simplicity in 279 search, remove "-", for strings, etc. 280 Also, concatenate the arg commands 281 For now, ignore nsize -- seems hard to search for anyway 282 """ 283 # Collect all of the args associated with a test 284 argStr=("" if 'args' not in testDict else testDict['args']) 285 if 'subtests' in testDict: 286 for stest in testDict["subtests"]: 287 sd=testDict[stest] 288 argStr=argStr+("" if 'args' not in sd else sd['args']) 289 290 # Now go through and cleanup 291 argStr=re.sub('{{(.*?)}}',"",argStr) 292 argStr=re.sub('-'," ",argStr) 293 for digit in string.digits: argStr=re.sub(digit," ",argStr) 294 argStr=re.sub(r"\.","",argStr) 295 argStr=re.sub(",","",argStr) 296 argStr=re.sub(r'\+',' ',argStr) 297 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 298 return argStr.strip() 299 300 def addToSources(self,exfile,rpath,srcDict): 301 """ 302 Put into data structure that allows easy generation of makefile 303 """ 304 pkg=rpath.split(os.path.sep)[0] 305 relpfile=os.path.join(rpath,exfile) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 if pkg not in self.sources: return 309 self.sources[pkg][lang]['srcs'].append(relpfile) 310 self.sources[pkg][lang][relpfile] = [] 311 if 'depends' in srcDict: 312 depSrcList=srcDict['depends'].split() 313 for depSrc in depSrcList: 314 depObj = getlangsplit(depSrc)+'.o' 315 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 316 317 # In gmakefile, ${TESTDIR} var specifies the object compilation 318 testsdir=rpath+"/" 319 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 320 self.objects[pkg].append(objfile) 321 return 322 323 def addToTests(self,test,rpath,exfile,execname,testDict): 324 """ 325 Put into data structure that allows easy generation of makefile 326 Organized by languages to allow testing of languages 327 """ 328 pkg=rpath.split("/")[0] 329 nmtest=os.path.join(rpath,test) 330 lang=self.getLanguage(exfile) 331 if not lang: return 332 if pkg not in self.tests: return 333 self.tests[pkg][lang][nmtest]={} 334 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 335 self.tests[pkg][lang][nmtest]['exec']=execname 336 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 337 return 338 339 def getExecname(self,exfile,rpath): 340 """ 341 Generate bash script using template found next to this file. 342 This file is read in at constructor time to avoid file I/O 343 """ 344 if self.single_ex: 345 execname=rpath.split("/")[1]+"-ex" 346 else: 347 execname=getlangsplit(exfile) 348 return execname 349 350 def getSubstVars(self,testDict,rpath,testname): 351 """ 352 Create a dictionary with all of the variables that get substituted 353 into the template commands found in example_template.py 354 """ 355 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 356 if 'nsize' not in testDict: testDict['nsize'] = '1' 357 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 358 subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'} 359 360 # Now do other variables 361 subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env'])) 362 subst['execname']=testDict['execname'] 363 subst['error']='' 364 if 'filter' in testDict: 365 if testDict['filter'].startswith("Error:"): 366 subst['error']="Error" 367 subst['filter']=testDict['filter'].lstrip("Error:") 368 else: 369 subst['filter']=testDict['filter'] 370 371 # Others 372 subst['subargs']='' # Default. For variables override 373 subst['srcdir']=os.path.join(self.srcdir, rpath) 374 subst['label_suffix']='' 375 subst['comments']="\n#".join(subst['comments'].split("\n")) 376 if subst['comments']: subst['comments']="#"+subst['comments'] 377 subst['executable']="../"+subst['execname'] 378 subst['testroot']=self.testroot_dir 379 subst['testname']=testname 380 dp = self.conf.get('DATAFILESPATH','') 381 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 382 383 # This is used to label some matrices 384 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 385 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 386 387 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 388 389 #Conf vars 390 if self.petsc_arch.find('valgrind')>=0: 391 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 392 else: 393 subst['mpiexec']=self.conf['MPIEXEC'] 394 subst['mpiexec_tail']=self.conf['MPIEXEC_TAIL'] 395 subst['pkg_name']=self.pkg_name 396 subst['pkg_dir']=self.pkg_dir 397 subst['pkg_arch']=self.pkg_arch 398 subst['CONFIG_DIR']=thisscriptdir 399 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 400 subst['diff']=self.conf['DIFF'] 401 subst['rm']=self.conf['RM'] 402 subst['grep']=self.conf['GREP'] 403 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 404 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 405 406 # Output file is special because of subtests override 407 defroot = testparse.getDefaultOutputFileRoot(testname) 408 if 'output_file' not in testDict: 409 subst['output_file']="output/"+defroot+".out" 410 subst['redirect_file']=defroot+".tmp" 411 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 412 413 # Add in the full path here. 414 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 415 416 subst['regexes']={} 417 for subkey in subst: 418 if subkey=='regexes': continue 419 if not isinstance(subst[subkey],str): continue 420 patt="@"+subkey.upper()+"@" 421 subst['regexes'][subkey]=re.compile(patt) 422 423 return subst 424 425 def _substVars(self,subst,origStr): 426 """ 427 Substitute variables 428 """ 429 Str=origStr 430 for subkey, subvalue in subst.items(): 431 if subkey=='regexes': continue 432 if not isinstance(subvalue,str): continue 433 if subkey.upper() not in Str: continue 434 Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str) 435 return Str 436 437 def getCmds(self,subst,i, debug=False): 438 """ 439 Generate bash script using template found next to this file. 440 This file is read in at constructor time to avoid file I/O 441 """ 442 nindnt=i # the start and has to be consistent with below 443 cmdindnt=self.indent*nindnt 444 cmdLines="" 445 446 # MPI is the default -- but we have a few odd commands 447 if subst['temporaries']: 448 if '*' in subst['temporaries']: 449 raise RuntimeError('{}/{}: list of temporary files to remove may not include wildcards'.format(subst['srcdir'], subst['execname'])) 450 cmd=cmdindnt+self._substVars(subst,example_template.preclean) 451 cmdLines+=cmd+"\n" 452 if not subst['command']: 453 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 454 else: 455 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 456 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 457 458 cmdLines+=cmdindnt+'if test $res = 0; then\n' 459 diffindnt=self.indent*(nindnt+1) 460 461 # Do some checks on existence of output_file and alt files 462 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 463 if not subst['TODO']: 464 print("Warning: "+subst['output_file']+" not found.") 465 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 466 467 # altlist always has output_file 468 if len(altlist)==1: 469 cmd=diffindnt+self._substVars(subst,example_template.difftest) 470 else: 471 if debug: print("Found alt files: ",altlist) 472 # Have to do it by hand a bit because of variable number of alt files 473 rf=subst['redirect_file'] 474 cmd=diffindnt+example_template.difftest.split('@')[0] 475 for i in range(len(altlist)): 476 af=altlist[i] 477 cmd+=af+' '+rf 478 if i!=len(altlist)-1: 479 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 480 cmd+=' || ${diff_exe} ' 481 else: 482 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 483 cmd+=subst['label_suffix']+' ""' # Quotes are painful 484 cmdLines+=cmd+"\n" 485 cmdLines+=cmdindnt+'else\n' 486 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 487 cmdLines+=cmdindnt+'fi\n' 488 return cmdLines 489 490 def _writeTodoSkip(self,fh,tors,reasons,footer): 491 """ 492 Write out the TODO and SKIP lines in the file 493 The TODO or SKIP variable, tors, should be lower case 494 """ 495 TORS=tors.upper() 496 template=eval("example_template."+tors+"line") 497 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 498 tab = '' 499 if reasons: 500 fh.write('if ! $force; then\n') 501 tab = tab + ' ' 502 if reasons == ["Requires DATAFILESPATH"]: 503 # The only reason not to run is DATAFILESPATH, which we check at run-time 504 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 505 tab = tab + ' ' 506 if reasons: 507 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 508 fh.write(tab+footer+"\n") 509 fh.write(tab+"exit\n") 510 if reasons == ["Requires DATAFILESPATH"]: 511 fh.write(' fi\n') 512 if reasons: 513 fh.write('fi\n') 514 fh.write('\n\n') 515 return 516 517 def getLoopVarsHead(self,loopVars,i,usedVars={}): 518 """ 519 Generate a nicely indented string with the format loops 520 Here is what the data structure looks like 521 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 522 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 523 loopVars['subargs']['pc_type']=["j","cholesky sor"] 524 """ 525 outstr=''; indnt=self.indent 526 527 for key in loopVars: 528 for var in loopVars[key]['varlist']: 529 varval=loopVars[key][var] 530 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 531 outstr += "\n\n" 532 533 for key in loopVars: 534 for var in loopVars[key]['varlist']: 535 varval=loopVars[key][var] 536 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 537 i = i + 1 538 return (outstr,i) 539 540 def getLoopVarsFoot(self,loopVars,i): 541 outstr=''; indnt=self.indent 542 for key in loopVars: 543 for var in loopVars[key]['varlist']: 544 i = i - 1 545 outstr += indnt * i + "done\n" 546 return (outstr,i) 547 548 def genRunScript(self,testname,root,isRun,srcDict): 549 """ 550 Generate bash script using template found next to this file. 551 This file is read in at constructor time to avoid file I/O 552 """ 553 def opener(path,flags,*args,**kwargs): 554 kwargs.setdefault('mode',0o755) 555 return os.open(path,flags,*args,**kwargs) 556 557 # runscript_dir directory has to be consistent with gmakefile 558 testDict=srcDict[testname] 559 rpath=self.srcrelpath(root) 560 runscript_dir=os.path.join(self.testroot_dir,rpath) 561 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 562 with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh: 563 564 # Get variables to go into shell scripts. last time testDict used 565 subst=self.getSubstVars(testDict,rpath,testname) 566 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 567 if 'subtests' in testDict: 568 # The subtests inherit inDict, so we don't need top-level loops. 569 loopVars = {} 570 571 #Handle runfiles 572 for lfile in subst.get('localrunfiles','').split(): 573 install_files(os.path.join(root, lfile), 574 os.path.join(runscript_dir, os.path.dirname(lfile))) 575 # Check subtests for local runfiles 576 for stest in subst.get("subtests",[]): 577 for lfile in testDict[stest].get('localrunfiles','').split(): 578 install_files(os.path.join(root, lfile), 579 os.path.join(runscript_dir, os.path.dirname(lfile))) 580 581 # Now substitute the key variables into the header and footer 582 header=self._substVars(subst,example_template.header) 583 # The header is done twice to enable @...@ in header 584 header=self._substVars(subst,header) 585 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 586 587 # Start writing the file 588 fh.write(header+"\n") 589 590 # If there is a TODO or a SKIP then we do it before writing out the 591 # rest of the command (which is useful for working on the test) 592 # SKIP and TODO can be for the source file or for the runs 593 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 594 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 595 596 j=0 # for indentation 597 598 if loopVars: 599 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 600 if (loopHead): fh.write(loopHead+"\n") 601 602 # Subtests are special 603 allLoopVars=list(loopVars.keys()) 604 if 'subtests' in testDict: 605 substP=subst # Subtests can inherit args but be careful 606 k=0 # for label suffixes 607 for stest in testDict["subtests"]: 608 subst=substP.copy() 609 subst.update(testDict[stest]) 610 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 611 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 612 if sLoopVars: 613 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 614 allLoopVars+=list(sLoopVars.keys()) 615 fh.write(sLoopHead+"\n") 616 fh.write(self.getCmds(subst,j)+"\n") 617 if sLoopVars: 618 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 619 fh.write(sLoopFoot+"\n") 620 else: 621 fh.write(self.getCmds(subst,j)+"\n") 622 623 if loopVars: 624 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 625 fh.write(loopFoot+"\n") 626 627 fh.write(footer+"\n") 628 return 629 630 def genScriptsAndInfo(self,exfile,root,srcDict): 631 """ 632 Generate scripts from the source file, determine if built, etc. 633 For every test in the exfile with info in the srcDict: 634 1. Determine if it needs to be run for this arch 635 2. Generate the script 636 3. Generate the data needed to write out the makefile in a 637 convenient way 638 All tests are *always* run, but some may be SKIP'd per the TAP standard 639 """ 640 debug=False 641 rpath=self.srcrelpath(root) 642 execname=self.getExecname(exfile,rpath) 643 isBuilt=self._isBuilt(exfile,srcDict) 644 for test in srcDict.copy(): 645 if test in self.buildkeys: continue 646 isRun=self._isRun(srcDict[test]) 647 # if the next two lines are dropped all scripts are generating included the unneeded 648 # if the unneeded are generated when run they will skip their tests automatically 649 # not generating them saves setup time 650 allow = False 651 if 'SKIP' in srcDict[test]: 652 allow = srcDict[test]['SKIP'] in [['Requires DATAFILESPATH'], ['PETSC_HAVE_PYVISTA requirement not met']] 653 if not isRun and not allow: 654 del srcDict[test] 655 continue 656 if 'TODO' in srcDict[test]: 657 del srcDict[test] 658 continue 659 srcDict[test]['execname']=execname # Convenience in generating scripts 660 self.genRunScript(test,root,isRun,srcDict) 661 srcDict[test]['isrun']=isRun 662 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 663 664 # This adds to datastructure for building deps 665 if isBuilt: self.addToSources(exfile,rpath,srcDict) 666 return 667 668 def _isBuilt(self,exfile,srcDict): 669 """ 670 Determine if this file should be built. 671 """ 672 # Get the language based on file extension 673 srcDict['SKIP'] = [] 674 lang=self.getLanguage(exfile) 675 if (lang=="F" or lang=="F90"): 676 if not self.have_fortran: 677 srcDict["SKIP"].append("Fortran required for this test") 678 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 679 srcDict["SKIP"].append("Fortran f90freeform required for this test") 680 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 681 srcDict["SKIP"].append("CUDA required for this test") 682 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 683 srcDict["SKIP"].append("HIP required for this test") 684 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 685 srcDict["SKIP"].append("SYCL required for this test") 686 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 687 srcDict["SKIP"].append("KOKKOS required for this test") 688 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 689 srcDict["SKIP"].append("RAJA required for this test") 690 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 691 srcDict["SKIP"].append("C++ required for this test") 692 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 693 srcDict["SKIP"].append("C++ required for this test") 694 695 # Deprecated source files 696 if srcDict.get("TODO"): 697 return False 698 699 # isRun can work with srcDict to handle the requires 700 if "requires" in srcDict: 701 if srcDict["requires"]: 702 return self._isRun(srcDict) 703 704 return srcDict['SKIP'] == [] 705 706 707 def _isRun(self,testDict, debug=False): 708 """ 709 Based on the requirements listed in the src file and the petscconf.h 710 info, determine whether this test should be run or not. 711 """ 712 indent=" " 713 714 if 'SKIP' not in testDict: 715 testDict['SKIP'] = [] 716 # MPI requirements 717 if 'MPI_IS_MPIUNI' in self.conf: 718 if testDict.get('nsize', '1') != '1': 719 testDict['SKIP'].append("Parallel test with serial build") 720 721 # The requirements for the test are the sum of all the run subtests 722 if 'subtests' in testDict: 723 if 'requires' not in testDict: testDict['requires']="" 724 for stest in testDict['subtests']: 725 if 'requires' in testDict[stest]: 726 testDict['requires']+=" "+testDict[stest]['requires'] 727 if testDict[stest].get('nsize', '1') != '1': 728 testDict['SKIP'].append("Parallel test with serial build") 729 break 730 731 # Now go through all requirements 732 if 'requires' in testDict: 733 for requirement in testDict['requires'].split(): 734 requirement=requirement.strip() 735 if not requirement: continue 736 if debug: print(indent+"Requirement: ", requirement) 737 isNull=False 738 if requirement.startswith("!"): 739 requirement=requirement[1:]; isNull=True 740 # 32-bit vs 64-bit pointers 741 if requirement == "64bitptr": 742 if self.conf['PETSC_SIZEOF_VOID_P']==8: 743 if isNull: 744 testDict['SKIP'].append("not 64bit-ptr required") 745 continue 746 continue # Success 747 elif not isNull: 748 testDict['SKIP'].append("64bit-ptr required") 749 continue 750 # Precision requirement for reals 751 if requirement in self.precision_types: 752 if self.conf['PETSC_PRECISION']==requirement: 753 if isNull: 754 testDict['SKIP'].append("not "+requirement+" required") 755 continue 756 continue # Success 757 elif not isNull: 758 testDict['SKIP'].append(requirement+" required") 759 continue 760 # Precision requirement for ints 761 if requirement in self.integer_types: 762 if requirement=="int32": 763 if self.conf['PETSC_SIZEOF_INT']==4: 764 if isNull: 765 testDict['SKIP'].append("not int32 required") 766 continue 767 continue # Success 768 elif not isNull: 769 testDict['SKIP'].append("int32 required") 770 continue 771 if requirement=="int64": 772 if self.conf['PETSC_SIZEOF_INT']==8: 773 if isNull: 774 testDict['SKIP'].append("NOT int64 required") 775 continue 776 continue # Success 777 elif not isNull: 778 testDict['SKIP'].append("int64 required") 779 continue 780 if requirement.startswith("long"): 781 reqsize = int(requirement[4:])//8 782 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 783 if longsize==reqsize: 784 if isNull: 785 testDict['SKIP'].append("not %s required" % requirement) 786 continue 787 continue # Success 788 elif not isNull: 789 testDict['SKIP'].append("%s required" % requirement) 790 continue 791 # Datafilespath 792 if requirement=="datafilespath" and not isNull: 793 testDict['SKIP'].append("Requires DATAFILESPATH") 794 continue 795 # Defines -- not sure I have comments matching 796 if "defined(" in requirement.lower(): 797 reqdef=requirement.split("(")[1].split(")")[0] 798 if reqdef in self.conf: 799 if isNull: 800 testDict['SKIP'].append("Null requirement not met: "+requirement) 801 continue 802 continue # Success 803 elif not isNull: 804 testDict['SKIP'].append("Required: "+requirement) 805 continue 806 807 # Rest should be packages that we can just get from conf 808 if requirement in ["complex","debug"]: 809 petscconfvar="PETSC_USE_"+requirement.upper() 810 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 811 else: 812 petscconfvar="PETSC_HAVE_"+requirement.upper() 813 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 814 petsccv = self.conf.get(petscconfvar) 815 pkgcv = self.conf.get(pkgconfvar) 816 817 if petsccv or pkgcv: 818 if isNull: 819 if petsccv: 820 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 821 continue 822 else: 823 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 824 continue 825 continue # Success 826 elif not isNull: 827 if not petsccv and not pkgcv: 828 if debug: print("requirement not found: ", requirement) 829 if self.pkg_name == 'petsc': 830 testDict['SKIP'].append(petscconfvar+" requirement not met") 831 else: 832 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 833 continue 834 return testDict['SKIP'] == [] 835 836 def checkOutput(self,exfile,root,srcDict): 837 """ 838 Check and make sure the output files are in the output directory 839 """ 840 debug=False 841 rpath=self.srcrelpath(root) 842 for test in srcDict: 843 if test in self.buildkeys: continue 844 if debug: print(rpath, exfile, test) 845 if 'output_file' in srcDict[test]: 846 output_file=srcDict[test]['output_file'] 847 else: 848 defroot = testparse.getDefaultOutputFileRoot(test) 849 if 'TODO' in srcDict[test]: continue 850 output_file="output/"+defroot+".out" 851 852 fullout=os.path.join(root,output_file) 853 if debug: print("---> ",fullout) 854 if not os.path.exists(fullout): 855 self.missing_files.append(fullout) 856 857 return 858 859 def genPetscTests_summarize(self,dataDict): 860 """ 861 Required method to state what happened 862 """ 863 if not self.summarize: return 864 indent=" " 865 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 866 with open(fhname, "w") as fh: 867 for root in dataDict: 868 relroot=self.srcrelpath(root) 869 pkg=relroot.split("/")[1] 870 if not pkg in self.sources: continue 871 fh.write(relroot+"\n") 872 allSrcs=[] 873 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 874 for exfile in dataDict[root]: 875 # Basic information 876 rfile=os.path.join(relroot,exfile) 877 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 878 fh.write(indent+exfile+indent*4+builtStatus+"\n") 879 for test in dataDict[root][exfile]: 880 if test in self.buildkeys: continue 881 line=indent*2+test 882 fh.write(line+"\n") 883 # Looks nice to have the keys in order 884 #for key in dataDict[root][exfile][test]: 885 for key in "isrun abstracted nsize args requires script".split(): 886 if key not in dataDict[root][exfile][test]: continue 887 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 888 fh.write(line+"\n") 889 fh.write("\n") 890 fh.write("\n") 891 fh.write("\n") 892 return 893 894 def genPetscTests(self,root,dirs,files,dataDict): 895 """ 896 Go through and parse the source files in the directory to generate 897 the examples based on the metadata contained in the source files 898 """ 899 debug=False 900 901 data = {} 902 for exfile in files: 903 #TST: Until we replace files, still leaving the originals as is 904 #if not exfile.startswith("new_"+"ex"): continue 905 #if not exfile.startswith("ex"): continue 906 907 # Ignore emacs and other temporary files 908 if exfile.startswith((".", "#")) or exfile.endswith("~"): continue 909 # Only parse source files 910 ext=getlangext(exfile).lstrip('.').replace('.','_') 911 if ext not in LANGS: continue 912 913 # Convenience 914 fullex=os.path.join(root,exfile) 915 if self.verbose: print(' --> '+fullex) 916 data.update(testparse.parseTestFile(fullex,0)) 917 if exfile in data: 918 if self.check_output: 919 self.checkOutput(exfile,root,data[exfile]) 920 else: 921 self.genScriptsAndInfo(exfile,root,data[exfile]) 922 923 dataDict[root] = data 924 return 925 926 def walktree(self,top): 927 """ 928 Walk a directory tree, starting from 'top' 929 """ 930 if self.check_output: 931 print("Checking for missing output files") 932 self.missing_files=[] 933 934 # Goal of action is to fill this dictionary 935 dataDict={} 936 for root, dirs, files in os.walk(top, topdown=True): 937 dirs.sort() 938 files.sort() 939 if "/tests" not in root and "/tutorials" not in root: continue 940 if "dSYM" in root: continue 941 if "tutorials"+os.sep+"build" in root: continue 942 if os.path.basename(root.rstrip("/")) == 'output': continue 943 if self.verbose: print(root) 944 self.genPetscTests(root,dirs,files,dataDict) 945 946 # If checking output, report results 947 if self.check_output: 948 if self.missing_files: 949 for file in set(self.missing_files): # set uniqifies 950 print(file) 951 sys.exit(1) 952 953 # Now summarize this dictionary 954 if self.verbose: self.genPetscTests_summarize(dataDict) 955 return dataDict 956 957 def gen_gnumake(self, fd): 958 """ 959 Overwrite of the method in the base PETSc class 960 """ 961 def write(stem, srcs): 962 for lang in LANGS: 963 if srcs[lang]['srcs']: 964 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 965 for pkg in self.pkg_pkgs: 966 srcs = self.gen_pkg(pkg) 967 write('testsrcs-' + pkg, srcs) 968 # Handle dependencies 969 for lang in LANGS: 970 for exfile in srcs[lang]['srcs']: 971 if exfile in srcs[lang]: 972 ex='$(TESTDIR)/'+getlangsplit(exfile) 973 exfo=ex+'.o' 974 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 975 if deps: 976 # The executable literally depends on the object file because it is linked 977 fd.write(ex +": " + " ".join(deps) +'\n') 978 # The object file containing 'main' does not normally depend on other object 979 # files, but it does when it includes their modules. This dependency is 980 # overly blunt and could be reduced to only depend on object files for 981 # modules that are used, like "*f90aux.o". 982 fd.write(exfo +": " + " ".join(deps) +'\n') 983 984 return self.gendeps 985 986 def gen_pkg(self, pkg): 987 """ 988 Overwrite of the method in the base PETSc class 989 """ 990 return self.sources[pkg] 991 992 def write_gnumake(self, dataDict, output=None): 993 """ 994 Write out something similar to files from gmakegen.py 995 996 Test depends on script which also depends on source 997 file, but since I don't have a good way generating 998 acting on a single file (oops) just depend on 999 executable which in turn will depend on src file 1000 """ 1001 # Different options for how to set up the targets 1002 compileExecsFirst=False 1003 1004 # Open file 1005 with open(output, 'w') as fd: 1006 # Write out the sources 1007 gendeps = self.gen_gnumake(fd) 1008 1009 # Write out the tests and execname targets 1010 fd.write("\n#Tests and executables\n") # Delimiter 1011 1012 for pkg in self.pkg_pkgs: 1013 # These grab the ones that are built 1014 for lang in LANGS: 1015 testdeps=[] 1016 for ftest in self.tests[pkg][lang]: 1017 test=os.path.basename(ftest) 1018 basedir=os.path.dirname(ftest) 1019 testdeps.append(nameSpace(test,basedir)) 1020 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1021 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1022 1023 # test targets 1024 for ftest in self.tests[pkg][lang]: 1025 test=os.path.basename(ftest) 1026 basedir=os.path.dirname(ftest) 1027 testdir="${TESTDIR}/"+basedir+"/" 1028 nmtest=nameSpace(test,basedir) 1029 rundir=os.path.join(testdir,test) 1030 script=test+".sh" 1031 1032 # Deps 1033 exfile=self.tests[pkg][lang][ftest]['exfile'] 1034 fullex=os.path.join(self.srcdir,exfile) 1035 localexec=self.tests[pkg][lang][ftest]['exec'] 1036 execname=os.path.join(testdir,localexec) 1037 fullscript=os.path.join(testdir,script) 1038 tmpfile=os.path.join(testdir,test,test+".tmp") 1039 1040 # *.counts depends on the script and either executable (will 1041 # be run) or the example source file (SKIP or TODO) 1042 fd.write('%s.counts : %s %s' 1043 % (os.path.join('$(TESTDIR)/counts', nmtest), 1044 fullscript, 1045 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1046 ) 1047 if exfile in self.sources[pkg][lang]: 1048 for dep in self.sources[pkg][lang][exfile]: 1049 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1050 fd.write('\n') 1051 1052 # Now write the args: 1053 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1054 1055 return 1056 1057 def write_db(self, dataDict, testdir): 1058 """ 1059 Write out the dataDict into a pickle file 1060 """ 1061 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1062 pickle.dump(dataDict,fd) 1063 return 1064 1065def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1066 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1067 srcdir=None, testdir=None, check=False): 1068 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1069 testdir=os.path.normpath(testdir) 1070 if petsc_arch: 1071 petsc_arch=petsc_arch.rstrip(os.path.sep) 1072 if len(petsc_arch.split(os.path.sep))>1: 1073 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1074 output = os.path.join(testdir, 'testfiles') 1075 1076 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1077 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1078 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1079 testdir=testdir,check=check) 1080 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1081 if not pEx.check_output: 1082 pEx.write_gnumake(dataDict, output) 1083 pEx.write_db(dataDict, testdir) 1084 1085if __name__ == '__main__': 1086 import optparse 1087 parser = optparse.OptionParser() 1088 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1089 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1090 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1091 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1092 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1093 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1094 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1095 help='Check whether output files are in output director') 1096 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1097 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1098 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1099 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1100 1101 opts, extra_args = parser.parse_args() 1102 if extra_args: 1103 import sys 1104 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1105 exit(1) 1106 if opts.testdir is None: 1107 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1108 1109 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1110 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1111 verbose=opts.verbose, 1112 single_ex=opts.single_executable, srcdir=opts.srcdir, 1113 testdir=opts.testdir, check=opts.check_output) 1114