1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 # Special configuration for CI testing 103 if self.petsc_arch.find('valgrind') >= 0: 104 self.conf['PETSCTEST_VALGRIND']=1 105 106 if self.inInstallDir: 107 # Case 2 discussed above 108 # set PETSC_ARCH to install directory to allow script to work in both 109 dirlist=thisscriptdir.split(os.path.sep) 110 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 111 self.arch_dir=installdir 112 if self.srcdir is None: 113 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 114 else: 115 if petsc_arch == '': 116 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 117 # Case 1 discussed above 118 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 119 if self.srcdir is None: 120 self.srcdir=os.path.join(self.petsc_dir,'src') 121 122 self.testroot_dir=os.path.abspath(testdir) 123 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64 long32 long64".split() 131 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in self.pkg_pkgs: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.relpath(rdir,self.srcdir) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext = getlangext(srcfile) 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 191 if srcext in ".cpp".split(): langReq="cpp" 192 if srcext == ".cu": langReq="cu" 193 if srcext == ".c": langReq="c" 194 #if not langReq: print("ERROR: ", srcext, srcfile) 195 return langReq 196 197 def _getAltList(self,output_file,srcdir): 198 ''' Calculate AltList based on output file-- see 199 src/snes/tutorials/output/ex22*.out 200 ''' 201 altlist=[output_file] 202 basefile = getlangsplit(output_file) 203 for i in range(1,9): 204 altroot=basefile+"_alt" 205 if i > 1: altroot=altroot+"_"+str(i) 206 af=altroot+".out" 207 srcaf=os.path.join(srcdir,af) 208 fullaf=os.path.join(self.petsc_dir,srcaf) 209 if os.path.isfile(fullaf): altlist.append(srcaf) 210 211 return altlist 212 213 214 def _getLoopVars(self,inDict,testname, isSubtest=False): 215 """ 216 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 217 Return: 218 inDict['args']: -ksp_monitor 219 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 220 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 221 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 222 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 223 subst should be passed in instead of inDict 224 """ 225 loopVars={}; newargs=[] 226 lsuffix='+' 227 argregex = re.compile(' (?=-[a-zA-Z])') 228 from testparse import parseLoopArgs 229 for key in inDict: 230 if key in ('SKIP', 'regexes'): 231 continue 232 akey=('subargs' if key=='args' else key) # what to assign 233 if akey not in inDict: inDict[akey]='' 234 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 235 # Always generate a loop over nsize, even if there is only one value 236 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 237 keystr = str(inDict[key]) 238 varlist = [] 239 for varset in argregex.split(keystr): 240 if not varset.strip(): continue 241 if '{{' in varset: 242 keyvar,lvars,ftype=parseLoopArgs(varset) 243 if akey not in loopVars: loopVars[akey]={} 244 varlist.append(keyvar) 245 loopVars[akey][keyvar]=[keyvar,lvars] 246 if akey=='nsize': 247 if len(lvars.split()) > 1: 248 lsuffix += akey +'-${i' + keyvar + '}' 249 else: 250 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 251 lsuffix+=keyvar+'-${i' + keyvar + '}_' 252 else: 253 if key=='args': 254 newargs.append(varset.strip()) 255 if varlist: 256 loopVars[akey]['varlist']=varlist 257 258 # For subtests, args are always substituted in (not top level) 259 if isSubtest: 260 inDict['subargs'] += " "+" ".join(newargs) 261 inDict['args']='' 262 if 'label_suffix' in inDict: 263 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 264 else: 265 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 266 else: 267 if loopVars: 268 inDict['args'] = ' '.join(newargs) 269 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 270 return loopVars 271 272 def getArgLabel(self,testDict): 273 """ 274 In all of the arguments in the test dictionary, create a simple 275 string for searching within the makefile system. For simplicity in 276 search, remove "-", for strings, etc. 277 Also, concatenate the arg commands 278 For now, ignore nsize -- seems hard to search for anyway 279 """ 280 # Collect all of the args associated with a test 281 argStr=("" if 'args' not in testDict else testDict['args']) 282 if 'subtests' in testDict: 283 for stest in testDict["subtests"]: 284 sd=testDict[stest] 285 argStr=argStr+("" if 'args' not in sd else sd['args']) 286 287 # Now go through and cleanup 288 argStr=re.sub('{{(.*?)}}',"",argStr) 289 argStr=re.sub('-'," ",argStr) 290 for digit in string.digits: argStr=re.sub(digit," ",argStr) 291 argStr=re.sub("\.","",argStr) 292 argStr=re.sub(",","",argStr) 293 argStr=re.sub('\+',' ',argStr) 294 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 295 return argStr.strip() 296 297 def addToSources(self,exfile,rpath,srcDict): 298 """ 299 Put into data structure that allows easy generation of makefile 300 """ 301 pkg=rpath.split(os.path.sep)[0] 302 relpfile=os.path.join(rpath,exfile) 303 lang=self.getLanguage(exfile) 304 if not lang: return 305 if pkg not in self.sources: return 306 self.sources[pkg][lang]['srcs'].append(relpfile) 307 self.sources[pkg][lang][relpfile] = [] 308 if 'depends' in srcDict: 309 depSrcList=srcDict['depends'].split() 310 for depSrc in depSrcList: 311 depObj = getlangsplit(depSrc)+'.o' 312 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 313 314 # In gmakefile, ${TESTDIR} var specifies the object compilation 315 testsdir=rpath+"/" 316 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 317 self.objects[pkg].append(objfile) 318 return 319 320 def addToTests(self,test,rpath,exfile,execname,testDict): 321 """ 322 Put into data structure that allows easy generation of makefile 323 Organized by languages to allow testing of languages 324 """ 325 pkg=rpath.split("/")[0] 326 nmtest=os.path.join(rpath,test) 327 lang=self.getLanguage(exfile) 328 if not lang: return 329 if pkg not in self.tests: return 330 self.tests[pkg][lang][nmtest]={} 331 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 332 self.tests[pkg][lang][nmtest]['exec']=execname 333 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 334 return 335 336 def getExecname(self,exfile,rpath): 337 """ 338 Generate bash script using template found next to this file. 339 This file is read in at constructor time to avoid file I/O 340 """ 341 if self.single_ex: 342 execname=rpath.split("/")[1]+"-ex" 343 else: 344 execname=getlangsplit(exfile) 345 return execname 346 347 def getSubstVars(self,testDict,rpath,testname): 348 """ 349 Create a dictionary with all of the variables that get substituted 350 into the template commands found in example_template.py 351 """ 352 subst={} 353 354 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 355 if 'nsize' not in testDict: testDict['nsize'] = '1' 356 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 357 for ak in testparse.acceptedkeys: 358 if ak=='test': continue 359 subst[ak]=(testDict[ak] if ak in testDict else '') 360 361 # Now do other variables 362 subst['execname']=testDict['execname'] 363 subst['error']='' 364 if 'filter' in testDict: 365 if testDict['filter'].startswith("Error:"): 366 subst['error']="Error" 367 subst['filter']=testDict['filter'].lstrip("Error:") 368 else: 369 subst['filter']=testDict['filter'] 370 371 # Others 372 subst['subargs']='' # Default. For variables override 373 subst['srcdir']=os.path.join(self.srcdir, rpath) 374 subst['label_suffix']='' 375 subst['comments']="\n#".join(subst['comments'].split("\n")) 376 if subst['comments']: subst['comments']="#"+subst['comments'] 377 subst['exec']="../"+subst['execname'] 378 subst['testroot']=self.testroot_dir 379 subst['testname']=testname 380 dp = self.conf.get('DATAFILESPATH','') 381 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 382 383 # This is used to label some matrices 384 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 385 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 386 387 #Conf vars 388 if self.petsc_arch.find('valgrind')>=0: 389 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 390 else: 391 subst['mpiexec']=self.conf['MPIEXEC'] 392 subst['pkg_name']=self.pkg_name 393 subst['pkg_dir']=self.pkg_dir 394 subst['pkg_arch']=self.petsc_arch 395 subst['CONFIG_DIR']=thisscriptdir 396 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 397 subst['diff']=self.conf['DIFF'] 398 subst['rm']=self.conf['RM'] 399 subst['grep']=self.conf['GREP'] 400 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 401 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 402 403 # Output file is special because of subtests override 404 defroot = testparse.getDefaultOutputFileRoot(testname) 405 if 'output_file' not in testDict: 406 subst['output_file']="output/"+defroot+".out" 407 subst['redirect_file']=defroot+".tmp" 408 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 409 410 # Add in the full path here. 411 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 412 413 subst['regexes']={} 414 for subkey in subst: 415 if subkey=='regexes': continue 416 if not isinstance(subst[subkey],str): continue 417 patt="@"+subkey.upper()+"@" 418 subst['regexes'][subkey]=re.compile(patt) 419 420 return subst 421 422 def _substVars(self,subst,origStr): 423 """ 424 Substitute variables 425 """ 426 Str=origStr 427 for subkey in subst: 428 if subkey=='regexes': continue 429 if not isinstance(subst[subkey],str): continue 430 if subkey.upper() not in Str: continue 431 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 432 return Str 433 434 def getCmds(self,subst,i, debug=False): 435 """ 436 Generate bash script using template found next to this file. 437 This file is read in at constructor time to avoid file I/O 438 """ 439 nindnt=i # the start and has to be consistent with below 440 cmdindnt=self.indent*nindnt 441 cmdLines="" 442 443 # MPI is the default -- but we have a few odd commands 444 if not subst['command']: 445 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 446 else: 447 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 448 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 449 450 cmdLines+=cmdindnt+'if test $res = 0; then\n' 451 diffindnt=self.indent*(nindnt+1) 452 453 # Do some checks on existence of output_file and alt files 454 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 455 if not subst['TODO']: 456 print("Warning: "+subst['output_file']+" not found.") 457 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 458 459 # altlist always has output_file 460 if len(altlist)==1: 461 cmd=diffindnt+self._substVars(subst,example_template.difftest) 462 else: 463 if debug: print("Found alt files: ",altlist) 464 # Have to do it by hand a bit because of variable number of alt files 465 rf=subst['redirect_file'] 466 cmd=diffindnt+example_template.difftest.split('@')[0] 467 for i in range(len(altlist)): 468 af=altlist[i] 469 cmd+=af+' '+rf 470 if i!=len(altlist)-1: 471 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 472 cmd+=' || ${diff_exe} ' 473 else: 474 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 475 cmd+=subst['label_suffix']+' ""' # Quotes are painful 476 cmdLines+=cmd+"\n" 477 cmdLines+=cmdindnt+'else\n' 478 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 479 cmdLines+=cmdindnt+'fi\n' 480 return cmdLines 481 482 def _writeTodoSkip(self,fh,tors,reasons,footer): 483 """ 484 Write out the TODO and SKIP lines in the file 485 The TODO or SKIP variable, tors, should be lower case 486 """ 487 TORS=tors.upper() 488 template=eval("example_template."+tors+"line") 489 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 490 tab = '' 491 if reasons: 492 fh.write('if ! $force; then\n') 493 tab = tab + ' ' 494 if reasons == ["Requires DATAFILESPATH"]: 495 # The only reason not to run is DATAFILESPATH, which we check at run-time 496 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 497 tab = tab + ' ' 498 if reasons: 499 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 500 fh.write(tab+footer+"\n") 501 fh.write(tab+"exit\n") 502 if reasons == ["Requires DATAFILESPATH"]: 503 fh.write(' fi\n') 504 if reasons: 505 fh.write('fi\n') 506 fh.write('\n\n') 507 return 508 509 def getLoopVarsHead(self,loopVars,i,usedVars={}): 510 """ 511 Generate a nicely indented string with the format loops 512 Here is what the data structure looks like 513 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 514 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 515 loopVars['subargs']['pc_type']=["j","cholesky sor"] 516 """ 517 outstr=''; indnt=self.indent 518 519 for key in loopVars: 520 for var in loopVars[key]['varlist']: 521 varval=loopVars[key][var] 522 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 523 outstr += "\n\n" 524 525 for key in loopVars: 526 for var in loopVars[key]['varlist']: 527 varval=loopVars[key][var] 528 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 529 i = i + 1 530 return (outstr,i) 531 532 def getLoopVarsFoot(self,loopVars,i): 533 outstr=''; indnt=self.indent 534 for key in loopVars: 535 for var in loopVars[key]['varlist']: 536 i = i - 1 537 outstr += indnt * i + "done\n" 538 return (outstr,i) 539 540 def genRunScript(self,testname,root,isRun,srcDict): 541 """ 542 Generate bash script using template found next to this file. 543 This file is read in at constructor time to avoid file I/O 544 """ 545 # runscript_dir directory has to be consistent with gmakefile 546 testDict=srcDict[testname] 547 rpath=self.srcrelpath(root) 548 runscript_dir=os.path.join(self.testroot_dir,rpath) 549 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 550 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 551 552 # Get variables to go into shell scripts. last time testDict used 553 subst=self.getSubstVars(testDict,rpath,testname) 554 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 555 if 'subtests' in testDict: 556 # The subtests inherit inDict, so we don't need top-level loops. 557 loopVars = {} 558 559 #Handle runfiles 560 for lfile in subst.get('localrunfiles','').split(): 561 install_files(os.path.join(root, lfile), 562 os.path.join(runscript_dir, os.path.dirname(lfile))) 563 # Check subtests for local runfiles 564 for stest in subst.get("subtests",[]): 565 for lfile in testDict[stest].get('localrunfiles','').split(): 566 install_files(os.path.join(root, lfile), 567 os.path.join(runscript_dir, os.path.dirname(lfile))) 568 569 # Now substitute the key variables into the header and footer 570 header=self._substVars(subst,example_template.header) 571 # The header is done twice to enable @...@ in header 572 header=self._substVars(subst,header) 573 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 574 575 # Start writing the file 576 fh.write(header+"\n") 577 578 # If there is a TODO or a SKIP then we do it before writing out the 579 # rest of the command (which is useful for working on the test) 580 # SKIP and TODO can be for the source file or for the runs 581 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 582 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 583 584 j=0 # for indentation 585 586 if loopVars: 587 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 588 if (loopHead): fh.write(loopHead+"\n") 589 590 # Subtests are special 591 allLoopVars=list(loopVars.keys()) 592 if 'subtests' in testDict: 593 substP=subst # Subtests can inherit args but be careful 594 k=0 # for label suffixes 595 for stest in testDict["subtests"]: 596 subst=substP.copy() 597 subst.update(testDict[stest]) 598 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 599 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 600 if sLoopVars: 601 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 602 allLoopVars+=list(sLoopVars.keys()) 603 fh.write(sLoopHead+"\n") 604 fh.write(self.getCmds(subst,j)+"\n") 605 if sLoopVars: 606 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 607 fh.write(sLoopFoot+"\n") 608 else: 609 fh.write(self.getCmds(subst,j)+"\n") 610 611 if loopVars: 612 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 613 fh.write(loopFoot+"\n") 614 615 fh.write(footer+"\n") 616 617 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 618 #if '10_9' in testname: sys.exit() 619 return 620 621 def genScriptsAndInfo(self,exfile,root,srcDict): 622 """ 623 Generate scripts from the source file, determine if built, etc. 624 For every test in the exfile with info in the srcDict: 625 1. Determine if it needs to be run for this arch 626 2. Generate the script 627 3. Generate the data needed to write out the makefile in a 628 convenient way 629 All tests are *always* run, but some may be SKIP'd per the TAP standard 630 """ 631 debug=False 632 rpath=self.srcrelpath(root) 633 execname=self.getExecname(exfile,rpath) 634 isBuilt=self._isBuilt(exfile,srcDict) 635 for test in srcDict: 636 if test in self.buildkeys: continue 637 if debug: print(nameSpace(exfile,root), test) 638 srcDict[test]['execname']=execname # Convenience in generating scripts 639 isRun=self._isRun(srcDict[test]) 640 self.genRunScript(test,root,isRun,srcDict) 641 srcDict[test]['isrun']=isRun 642 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 643 644 # This adds to datastructure for building deps 645 if isBuilt: self.addToSources(exfile,rpath,srcDict) 646 return 647 648 def _isBuilt(self,exfile,srcDict): 649 """ 650 Determine if this file should be built. 651 """ 652 # Get the language based on file extension 653 srcDict['SKIP'] = [] 654 lang=self.getLanguage(exfile) 655 if (lang=="F" or lang=="F90"): 656 if not self.have_fortran: 657 srcDict["SKIP"].append("Fortran required for this test") 658 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 659 srcDict["SKIP"].append("Fortran f90freeform required for this test") 660 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 661 srcDict["SKIP"].append("CUDA required for this test") 662 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 663 srcDict["SKIP"].append("HIP required for this test") 664 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 665 srcDict["SKIP"].append("SYCL required for this test") 666 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 667 srcDict["SKIP"].append("KOKKOS required for this test") 668 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 669 srcDict["SKIP"].append("C++ required for this test") 670 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 671 srcDict["SKIP"].append("C++ required for this test") 672 673 # Deprecated source files 674 if srcDict.get("TODO"): 675 return False 676 677 # isRun can work with srcDict to handle the requires 678 if "requires" in srcDict: 679 if srcDict["requires"]: 680 return self._isRun(srcDict) 681 682 return srcDict['SKIP'] == [] 683 684 685 def _isRun(self,testDict, debug=False): 686 """ 687 Based on the requirements listed in the src file and the petscconf.h 688 info, determine whether this test should be run or not. 689 """ 690 indent=" " 691 692 if 'SKIP' not in testDict: 693 testDict['SKIP'] = [] 694 # MPI requirements 695 if 'MPI_IS_MPIUNI' in self.conf: 696 if testDict.get('nsize', '1') != '1': 697 testDict['SKIP'].append("Parallel test with serial build") 698 699 # The requirements for the test are the sum of all the run subtests 700 if 'subtests' in testDict: 701 if 'requires' not in testDict: testDict['requires']="" 702 for stest in testDict['subtests']: 703 if 'requires' in testDict[stest]: 704 testDict['requires']+=" "+testDict[stest]['requires'] 705 if testDict[stest].get('nsize', '1') != '1': 706 testDict['SKIP'].append("Parallel test with serial build") 707 break 708 709 # Now go through all requirements 710 if 'requires' in testDict: 711 for requirement in testDict['requires'].split(): 712 requirement=requirement.strip() 713 if not requirement: continue 714 if debug: print(indent+"Requirement: ", requirement) 715 isNull=False 716 if requirement.startswith("!"): 717 requirement=requirement[1:]; isNull=True 718 # Precision requirement for reals 719 if requirement in self.precision_types: 720 if self.conf['PETSC_PRECISION']==requirement: 721 if isNull: 722 testDict['SKIP'].append("not "+requirement+" required") 723 continue 724 continue # Success 725 elif not isNull: 726 testDict['SKIP'].append(requirement+" required") 727 continue 728 # Precision requirement for ints 729 if requirement in self.integer_types: 730 if requirement=="int32": 731 if self.conf['PETSC_SIZEOF_INT']==4: 732 if isNull: 733 testDict['SKIP'].append("not int32 required") 734 continue 735 continue # Success 736 elif not isNull: 737 testDict['SKIP'].append("int32 required") 738 continue 739 if requirement=="int64": 740 if self.conf['PETSC_SIZEOF_INT']==8: 741 if isNull: 742 testDict['SKIP'].append("NOT int64 required") 743 continue 744 continue # Success 745 elif not isNull: 746 testDict['SKIP'].append("int64 required") 747 continue 748 if requirement.startswith("long"): 749 reqsize = int(requirement[4:])//8 750 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 751 if longsize==reqsize: 752 if isNull: 753 testDict['SKIP'].append("not %s required" % requirement) 754 continue 755 continue # Success 756 elif not isNull: 757 testDict['SKIP'].append("%s required" % requirement) 758 continue 759 # Datafilespath 760 if requirement=="datafilespath" and not isNull: 761 testDict['SKIP'].append("Requires DATAFILESPATH") 762 continue 763 # Defines -- not sure I have comments matching 764 if "defined(" in requirement.lower(): 765 reqdef=requirement.split("(")[1].split(")")[0] 766 if reqdef in self.conf: 767 if isNull: 768 testDict['SKIP'].append("Null requirement not met: "+requirement) 769 continue 770 continue # Success 771 elif not isNull: 772 testDict['SKIP'].append("Required: "+requirement) 773 continue 774 775 # Rest should be packages that we can just get from conf 776 if requirement in ["complex","debug"]: 777 petscconfvar="PETSC_USE_"+requirement.upper() 778 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 779 else: 780 petscconfvar="PETSC_HAVE_"+requirement.upper() 781 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 782 petsccv = self.conf.get(petscconfvar) 783 pkgcv = self.conf.get(pkgconfvar) 784 785 if petsccv or pkgcv: 786 if isNull: 787 if petsccv: 788 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 789 continue 790 else: 791 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 792 continue 793 continue # Success 794 elif not isNull: 795 if not petsccv and not pkgcv: 796 if debug: print("requirement not found: ", requirement) 797 if self.pkg_name == 'petsc': 798 testDict['SKIP'].append(petscconfvar+" requirement not met") 799 else: 800 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 801 continue 802 return testDict['SKIP'] == [] 803 804 def checkOutput(self,exfile,root,srcDict): 805 """ 806 Check and make sure the output files are in the output directory 807 """ 808 debug=False 809 rpath=self.srcrelpath(root) 810 for test in srcDict: 811 if test in self.buildkeys: continue 812 if debug: print(rpath, exfile, test) 813 if 'output_file' in srcDict[test]: 814 output_file=srcDict[test]['output_file'] 815 else: 816 defroot = testparse.getDefaultOutputFileRoot(test) 817 if 'TODO' in srcDict[test]: continue 818 output_file="output/"+defroot+".out" 819 820 fullout=os.path.join(root,output_file) 821 if debug: print("---> ",fullout) 822 if not os.path.exists(fullout): 823 self.missing_files.append(fullout) 824 825 return 826 827 def genPetscTests_summarize(self,dataDict): 828 """ 829 Required method to state what happened 830 """ 831 if not self.summarize: return 832 indent=" " 833 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 834 with open(fhname, "w") as fh: 835 for root in dataDict: 836 relroot=self.srcrelpath(root) 837 pkg=relroot.split("/")[1] 838 fh.write(relroot+"\n") 839 allSrcs=[] 840 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 841 for exfile in dataDict[root]: 842 # Basic information 843 rfile=os.path.join(relroot,exfile) 844 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 845 fh.write(indent+exfile+indent*4+builtStatus+"\n") 846 for test in dataDict[root][exfile]: 847 if test in self.buildkeys: continue 848 line=indent*2+test 849 fh.write(line+"\n") 850 # Looks nice to have the keys in order 851 #for key in dataDict[root][exfile][test]: 852 for key in "isrun abstracted nsize args requires script".split(): 853 if key not in dataDict[root][exfile][test]: continue 854 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 855 fh.write(line+"\n") 856 fh.write("\n") 857 fh.write("\n") 858 fh.write("\n") 859 return 860 861 def genPetscTests(self,root,dirs,files,dataDict): 862 """ 863 Go through and parse the source files in the directory to generate 864 the examples based on the metadata contained in the source files 865 """ 866 debug=False 867 # Use examplesAnalyze to get what the makefles think are sources 868 #self.examplesAnalyze(root,dirs,files,anlzDict) 869 870 dataDict[root]={} 871 872 for exfile in files: 873 #TST: Until we replace files, still leaving the orginals as is 874 #if not exfile.startswith("new_"+"ex"): continue 875 #if not exfile.startswith("ex"): continue 876 877 # Ignore emacs and other temporary files 878 if exfile.startswith("."): continue 879 if exfile.startswith("#"): continue 880 if exfile.endswith("~"): continue 881 # Only parse source files 882 ext=getlangext(exfile).lstrip('.').replace('.','_') 883 if ext not in LANGS: continue 884 885 # Convenience 886 fullex=os.path.join(root,exfile) 887 if self.verbose: print(' --> '+fullex) 888 dataDict[root].update(testparse.parseTestFile(fullex,0)) 889 if exfile in dataDict[root]: 890 if not self.check_output: 891 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 892 else: 893 self.checkOutput(exfile,root,dataDict[root][exfile]) 894 895 return 896 897 def walktree(self,top): 898 """ 899 Walk a directory tree, starting from 'top' 900 """ 901 if self.check_output: 902 print("Checking for missing output files") 903 self.missing_files=[] 904 905 # Goal of action is to fill this dictionary 906 dataDict={} 907 for root, dirs, files in os.walk(top, topdown=True): 908 dirs.sort() 909 files.sort() 910 if "/tests" not in root and "/tutorials" not in root: continue 911 if "dSYM" in root: continue 912 if "tutorials"+os.sep+"build" in root: continue 913 if os.path.basename(root.rstrip("/")) == 'output': continue 914 if self.verbose: print(root) 915 self.genPetscTests(root,dirs,files,dataDict) 916 917 # If checking output, report results 918 if self.check_output: 919 if self.missing_files: 920 for file in set(self.missing_files): # set uniqifies 921 print(file) 922 sys.exit(1) 923 924 # Now summarize this dictionary 925 if self.verbose: self.genPetscTests_summarize(dataDict) 926 return dataDict 927 928 def gen_gnumake(self, fd): 929 """ 930 Overwrite of the method in the base PETSc class 931 """ 932 def write(stem, srcs): 933 for lang in LANGS: 934 if srcs[lang]['srcs']: 935 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 936 for pkg in self.pkg_pkgs: 937 srcs = self.gen_pkg(pkg) 938 write('testsrcs-' + pkg, srcs) 939 # Handle dependencies 940 for lang in LANGS: 941 for exfile in srcs[lang]['srcs']: 942 if exfile in srcs[lang]: 943 ex='$(TESTDIR)/'+getlangsplit(exfile) 944 exfo=ex+'.o' 945 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 946 if deps: 947 # The executable literally depends on the object file because it is linked 948 fd.write(ex +": " + " ".join(deps) +'\n') 949 # The object file containing 'main' does not normally depend on other object 950 # files, but it does when it includes their modules. This dependency is 951 # overly blunt and could be reduced to only depend on object files for 952 # modules that are used, like "*f90aux.o". 953 fd.write(exfo +": " + " ".join(deps) +'\n') 954 955 return self.gendeps 956 957 def gen_pkg(self, pkg): 958 """ 959 Overwrite of the method in the base PETSc class 960 """ 961 return self.sources[pkg] 962 963 def write_gnumake(self, dataDict, output=None): 964 """ 965 Write out something similar to files from gmakegen.py 966 967 Test depends on script which also depends on source 968 file, but since I don't have a good way generating 969 acting on a single file (oops) just depend on 970 executable which in turn will depend on src file 971 """ 972 # Different options for how to set up the targets 973 compileExecsFirst=False 974 975 # Open file 976 with open(output, 'w') as fd: 977 # Write out the sources 978 gendeps = self.gen_gnumake(fd) 979 980 # Write out the tests and execname targets 981 fd.write("\n#Tests and executables\n") # Delimiter 982 983 for pkg in self.pkg_pkgs: 984 # These grab the ones that are built 985 for lang in LANGS: 986 testdeps=[] 987 for ftest in self.tests[pkg][lang]: 988 test=os.path.basename(ftest) 989 basedir=os.path.dirname(ftest) 990 testdeps.append(nameSpace(test,basedir)) 991 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 992 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 993 994 # test targets 995 for ftest in self.tests[pkg][lang]: 996 test=os.path.basename(ftest) 997 basedir=os.path.dirname(ftest) 998 testdir="${TESTDIR}/"+basedir+"/" 999 nmtest=nameSpace(test,basedir) 1000 rundir=os.path.join(testdir,test) 1001 script=test+".sh" 1002 1003 # Deps 1004 exfile=self.tests[pkg][lang][ftest]['exfile'] 1005 fullex=os.path.join(self.srcdir,exfile) 1006 localexec=self.tests[pkg][lang][ftest]['exec'] 1007 execname=os.path.join(testdir,localexec) 1008 fullscript=os.path.join(testdir,script) 1009 tmpfile=os.path.join(testdir,test,test+".tmp") 1010 1011 # *.counts depends on the script and either executable (will 1012 # be run) or the example source file (SKIP or TODO) 1013 fd.write('%s.counts : %s %s' 1014 % (os.path.join('$(TESTDIR)/counts', nmtest), 1015 fullscript, 1016 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1017 ) 1018 if exfile in self.sources[pkg][lang]: 1019 for dep in self.sources[pkg][lang][exfile]: 1020 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1021 fd.write('\n') 1022 1023 # Now write the args: 1024 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1025 1026 return 1027 1028 def write_db(self, dataDict, testdir): 1029 """ 1030 Write out the dataDict into a pickle file 1031 """ 1032 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1033 pickle.dump(dataDict,fd) 1034 return 1035 1036def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1037 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1038 srcdir=None, testdir=None, check=False): 1039 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1040 testdir=os.path.normpath(testdir) 1041 if petsc_arch: 1042 petsc_arch=petsc_arch.rstrip(os.path.sep) 1043 if len(petsc_arch.split(os.path.sep))>1: 1044 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1045 output = os.path.join(testdir, 'testfiles') 1046 1047 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1048 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1049 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1050 testdir=testdir,check=check) 1051 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1052 if not pEx.check_output: 1053 pEx.write_gnumake(dataDict, output) 1054 pEx.write_db(dataDict, testdir) 1055 1056if __name__ == '__main__': 1057 import optparse 1058 parser = optparse.OptionParser() 1059 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1060 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1061 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1062 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1063 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1064 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1065 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1066 help='Check whether output files are in output director') 1067 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1068 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1069 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1070 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1071 1072 opts, extra_args = parser.parse_args() 1073 if extra_args: 1074 import sys 1075 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1076 exit(1) 1077 if opts.testdir is None: 1078 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1079 1080 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1081 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1082 verbose=opts.verbose, 1083 single_ex=opts.single_executable, srcdir=opts.srcdir, 1084 testdir=opts.testdir, check=opts.check_output) 1085