1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 # Special configuration for CI testing 103 if self.petsc_arch.find('valgrind') >= 0: 104 self.conf['PETSCTEST_VALGRIND']=1 105 106 if self.inInstallDir: 107 # Case 2 discussed above 108 # set PETSC_ARCH to install directory to allow script to work in both 109 dirlist=thisscriptdir.split(os.path.sep) 110 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 111 self.arch_dir=installdir 112 if self.srcdir is None: 113 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 114 else: 115 if petsc_arch == '': 116 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 117 # Case 1 discussed above 118 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 119 if self.srcdir is None: 120 self.srcdir=os.path.join(self.petsc_dir,'src') 121 122 self.testroot_dir=os.path.abspath(testdir) 123 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64 long32 long64".split() 131 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in self.pkg_pkgs: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.relpath(rdir,self.srcdir) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext = getlangext(srcfile) 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 191 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 192 if srcext in ".cpp".split(): langReq="cpp" 193 if srcext == ".cu": langReq="cu" 194 if srcext == ".c": langReq="c" 195 #if not langReq: print("ERROR: ", srcext, srcfile) 196 return langReq 197 198 def _getAltList(self,output_file,srcdir): 199 ''' Calculate AltList based on output file-- see 200 src/snes/tutorials/output/ex22*.out 201 ''' 202 altlist=[output_file] 203 basefile = getlangsplit(output_file) 204 for i in range(1,9): 205 altroot=basefile+"_alt" 206 if i > 1: altroot=altroot+"_"+str(i) 207 af=altroot+".out" 208 srcaf=os.path.join(srcdir,af) 209 fullaf=os.path.join(self.petsc_dir,srcaf) 210 if os.path.isfile(fullaf): altlist.append(srcaf) 211 212 return altlist 213 214 215 def _getLoopVars(self,inDict,testname, isSubtest=False): 216 """ 217 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 218 Return: 219 inDict['args']: -ksp_monitor 220 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 221 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 222 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 223 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 224 subst should be passed in instead of inDict 225 """ 226 loopVars={}; newargs=[] 227 lsuffix='+' 228 argregex = re.compile(' (?=-[a-zA-Z])') 229 from testparse import parseLoopArgs 230 for key in inDict: 231 if key in ('SKIP', 'regexes'): 232 continue 233 akey=('subargs' if key=='args' else key) # what to assign 234 if akey not in inDict: inDict[akey]='' 235 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 236 # Always generate a loop over nsize, even if there is only one value 237 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 238 keystr = str(inDict[key]) 239 varlist = [] 240 for varset in argregex.split(keystr): 241 if not varset.strip(): continue 242 if '{{' in varset: 243 keyvar,lvars,ftype=parseLoopArgs(varset) 244 if akey not in loopVars: loopVars[akey]={} 245 varlist.append(keyvar) 246 loopVars[akey][keyvar]=[keyvar,lvars] 247 if akey=='nsize': 248 if len(lvars.split()) > 1: 249 lsuffix += akey +'-${i' + keyvar + '}' 250 else: 251 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 252 lsuffix+=keyvar+'-${i' + keyvar + '}_' 253 else: 254 if key=='args': 255 newargs.append(varset.strip()) 256 if varlist: 257 loopVars[akey]['varlist']=varlist 258 259 # For subtests, args are always substituted in (not top level) 260 if isSubtest: 261 inDict['subargs'] += " "+" ".join(newargs) 262 inDict['args']='' 263 if 'label_suffix' in inDict: 264 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 265 else: 266 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 267 else: 268 if loopVars: 269 inDict['args'] = ' '.join(newargs) 270 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 271 return loopVars 272 273 def getArgLabel(self,testDict): 274 """ 275 In all of the arguments in the test dictionary, create a simple 276 string for searching within the makefile system. For simplicity in 277 search, remove "-", for strings, etc. 278 Also, concatenate the arg commands 279 For now, ignore nsize -- seems hard to search for anyway 280 """ 281 # Collect all of the args associated with a test 282 argStr=("" if 'args' not in testDict else testDict['args']) 283 if 'subtests' in testDict: 284 for stest in testDict["subtests"]: 285 sd=testDict[stest] 286 argStr=argStr+("" if 'args' not in sd else sd['args']) 287 288 # Now go through and cleanup 289 argStr=re.sub('{{(.*?)}}',"",argStr) 290 argStr=re.sub('-'," ",argStr) 291 for digit in string.digits: argStr=re.sub(digit," ",argStr) 292 argStr=re.sub("\.","",argStr) 293 argStr=re.sub(",","",argStr) 294 argStr=re.sub('\+',' ',argStr) 295 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 296 return argStr.strip() 297 298 def addToSources(self,exfile,rpath,srcDict): 299 """ 300 Put into data structure that allows easy generation of makefile 301 """ 302 pkg=rpath.split(os.path.sep)[0] 303 relpfile=os.path.join(rpath,exfile) 304 lang=self.getLanguage(exfile) 305 if not lang: return 306 if pkg not in self.sources: return 307 self.sources[pkg][lang]['srcs'].append(relpfile) 308 self.sources[pkg][lang][relpfile] = [] 309 if 'depends' in srcDict: 310 depSrcList=srcDict['depends'].split() 311 for depSrc in depSrcList: 312 depObj = getlangsplit(depSrc)+'.o' 313 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 314 315 # In gmakefile, ${TESTDIR} var specifies the object compilation 316 testsdir=rpath+"/" 317 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 318 self.objects[pkg].append(objfile) 319 return 320 321 def addToTests(self,test,rpath,exfile,execname,testDict): 322 """ 323 Put into data structure that allows easy generation of makefile 324 Organized by languages to allow testing of languages 325 """ 326 pkg=rpath.split("/")[0] 327 nmtest=os.path.join(rpath,test) 328 lang=self.getLanguage(exfile) 329 if not lang: return 330 if pkg not in self.tests: return 331 self.tests[pkg][lang][nmtest]={} 332 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 333 self.tests[pkg][lang][nmtest]['exec']=execname 334 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 335 return 336 337 def getExecname(self,exfile,rpath): 338 """ 339 Generate bash script using template found next to this file. 340 This file is read in at constructor time to avoid file I/O 341 """ 342 if self.single_ex: 343 execname=rpath.split("/")[1]+"-ex" 344 else: 345 execname=getlangsplit(exfile) 346 return execname 347 348 def getSubstVars(self,testDict,rpath,testname): 349 """ 350 Create a dictionary with all of the variables that get substituted 351 into the template commands found in example_template.py 352 """ 353 subst={} 354 355 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 356 if 'nsize' not in testDict: testDict['nsize'] = '1' 357 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 358 for ak in testparse.acceptedkeys: 359 if ak=='test': continue 360 subst[ak]=(testDict[ak] if ak in testDict else '') 361 362 # Now do other variables 363 subst['execname']=testDict['execname'] 364 subst['error']='' 365 if 'filter' in testDict: 366 if testDict['filter'].startswith("Error:"): 367 subst['error']="Error" 368 subst['filter']=testDict['filter'].lstrip("Error:") 369 else: 370 subst['filter']=testDict['filter'] 371 372 # Others 373 subst['subargs']='' # Default. For variables override 374 subst['srcdir']=os.path.join(self.srcdir, rpath) 375 subst['label_suffix']='' 376 subst['comments']="\n#".join(subst['comments'].split("\n")) 377 if subst['comments']: subst['comments']="#"+subst['comments'] 378 subst['exec']="../"+subst['execname'] 379 subst['testroot']=self.testroot_dir 380 subst['testname']=testname 381 dp = self.conf.get('DATAFILESPATH','') 382 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 383 384 # This is used to label some matrices 385 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 386 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 387 388 if 'PETSC_OPTIONS' in os.environ: 389 subst['petsc_options']=os.environ['PETSC_OPTIONS'] 390 else: 391 subst['petsc_options']='' 392 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 393 394 #Conf vars 395 if self.petsc_arch.find('valgrind')>=0: 396 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 397 else: 398 subst['mpiexec']=self.conf['MPIEXEC'] 399 subst['pkg_name']=self.pkg_name 400 subst['pkg_dir']=self.pkg_dir 401 subst['pkg_arch']=self.petsc_arch 402 subst['CONFIG_DIR']=thisscriptdir 403 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 404 subst['diff']=self.conf['DIFF'] 405 subst['rm']=self.conf['RM'] 406 subst['grep']=self.conf['GREP'] 407 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 408 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 409 410 # Output file is special because of subtests override 411 defroot = testparse.getDefaultOutputFileRoot(testname) 412 if 'output_file' not in testDict: 413 subst['output_file']="output/"+defroot+".out" 414 subst['redirect_file']=defroot+".tmp" 415 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 416 417 # Add in the full path here. 418 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 419 420 subst['regexes']={} 421 for subkey in subst: 422 if subkey=='regexes': continue 423 if not isinstance(subst[subkey],str): continue 424 patt="@"+subkey.upper()+"@" 425 subst['regexes'][subkey]=re.compile(patt) 426 427 return subst 428 429 def _substVars(self,subst,origStr): 430 """ 431 Substitute variables 432 """ 433 Str=origStr 434 for subkey in subst: 435 if subkey=='regexes': continue 436 if not isinstance(subst[subkey],str): continue 437 if subkey.upper() not in Str: continue 438 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 439 return Str 440 441 def getCmds(self,subst,i, debug=False): 442 """ 443 Generate bash script using template found next to this file. 444 This file is read in at constructor time to avoid file I/O 445 """ 446 nindnt=i # the start and has to be consistent with below 447 cmdindnt=self.indent*nindnt 448 cmdLines="" 449 450 # MPI is the default -- but we have a few odd commands 451 if not subst['command']: 452 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 453 else: 454 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 455 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 456 457 cmdLines+=cmdindnt+'if test $res = 0; then\n' 458 diffindnt=self.indent*(nindnt+1) 459 460 # Do some checks on existence of output_file and alt files 461 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 462 if not subst['TODO']: 463 print("Warning: "+subst['output_file']+" not found.") 464 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 465 466 # altlist always has output_file 467 if len(altlist)==1: 468 cmd=diffindnt+self._substVars(subst,example_template.difftest) 469 else: 470 if debug: print("Found alt files: ",altlist) 471 # Have to do it by hand a bit because of variable number of alt files 472 rf=subst['redirect_file'] 473 cmd=diffindnt+example_template.difftest.split('@')[0] 474 for i in range(len(altlist)): 475 af=altlist[i] 476 cmd+=af+' '+rf 477 if i!=len(altlist)-1: 478 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 479 cmd+=' || ${diff_exe} ' 480 else: 481 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 482 cmd+=subst['label_suffix']+' ""' # Quotes are painful 483 cmdLines+=cmd+"\n" 484 cmdLines+=cmdindnt+'else\n' 485 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 486 cmdLines+=cmdindnt+'fi\n' 487 return cmdLines 488 489 def _writeTodoSkip(self,fh,tors,reasons,footer): 490 """ 491 Write out the TODO and SKIP lines in the file 492 The TODO or SKIP variable, tors, should be lower case 493 """ 494 TORS=tors.upper() 495 template=eval("example_template."+tors+"line") 496 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 497 tab = '' 498 if reasons: 499 fh.write('if ! $force; then\n') 500 tab = tab + ' ' 501 if reasons == ["Requires DATAFILESPATH"]: 502 # The only reason not to run is DATAFILESPATH, which we check at run-time 503 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 504 tab = tab + ' ' 505 if reasons: 506 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 507 fh.write(tab+footer+"\n") 508 fh.write(tab+"exit\n") 509 if reasons == ["Requires DATAFILESPATH"]: 510 fh.write(' fi\n') 511 if reasons: 512 fh.write('fi\n') 513 fh.write('\n\n') 514 return 515 516 def getLoopVarsHead(self,loopVars,i,usedVars={}): 517 """ 518 Generate a nicely indented string with the format loops 519 Here is what the data structure looks like 520 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 521 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 522 loopVars['subargs']['pc_type']=["j","cholesky sor"] 523 """ 524 outstr=''; indnt=self.indent 525 526 for key in loopVars: 527 for var in loopVars[key]['varlist']: 528 varval=loopVars[key][var] 529 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 530 outstr += "\n\n" 531 532 for key in loopVars: 533 for var in loopVars[key]['varlist']: 534 varval=loopVars[key][var] 535 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 536 i = i + 1 537 return (outstr,i) 538 539 def getLoopVarsFoot(self,loopVars,i): 540 outstr=''; indnt=self.indent 541 for key in loopVars: 542 for var in loopVars[key]['varlist']: 543 i = i - 1 544 outstr += indnt * i + "done\n" 545 return (outstr,i) 546 547 def genRunScript(self,testname,root,isRun,srcDict): 548 """ 549 Generate bash script using template found next to this file. 550 This file is read in at constructor time to avoid file I/O 551 """ 552 # runscript_dir directory has to be consistent with gmakefile 553 testDict=srcDict[testname] 554 rpath=self.srcrelpath(root) 555 runscript_dir=os.path.join(self.testroot_dir,rpath) 556 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 557 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 558 559 # Get variables to go into shell scripts. last time testDict used 560 subst=self.getSubstVars(testDict,rpath,testname) 561 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 562 if 'subtests' in testDict: 563 # The subtests inherit inDict, so we don't need top-level loops. 564 loopVars = {} 565 566 #Handle runfiles 567 for lfile in subst.get('localrunfiles','').split(): 568 install_files(os.path.join(root, lfile), 569 os.path.join(runscript_dir, os.path.dirname(lfile))) 570 # Check subtests for local runfiles 571 for stest in subst.get("subtests",[]): 572 for lfile in testDict[stest].get('localrunfiles','').split(): 573 install_files(os.path.join(root, lfile), 574 os.path.join(runscript_dir, os.path.dirname(lfile))) 575 576 # Now substitute the key variables into the header and footer 577 header=self._substVars(subst,example_template.header) 578 # The header is done twice to enable @...@ in header 579 header=self._substVars(subst,header) 580 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 581 582 # Start writing the file 583 fh.write(header+"\n") 584 585 # If there is a TODO or a SKIP then we do it before writing out the 586 # rest of the command (which is useful for working on the test) 587 # SKIP and TODO can be for the source file or for the runs 588 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 589 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 590 591 j=0 # for indentation 592 593 if loopVars: 594 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 595 if (loopHead): fh.write(loopHead+"\n") 596 597 # Subtests are special 598 allLoopVars=list(loopVars.keys()) 599 if 'subtests' in testDict: 600 substP=subst # Subtests can inherit args but be careful 601 k=0 # for label suffixes 602 for stest in testDict["subtests"]: 603 subst=substP.copy() 604 subst.update(testDict[stest]) 605 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 606 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 607 if sLoopVars: 608 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 609 allLoopVars+=list(sLoopVars.keys()) 610 fh.write(sLoopHead+"\n") 611 fh.write(self.getCmds(subst,j)+"\n") 612 if sLoopVars: 613 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 614 fh.write(sLoopFoot+"\n") 615 else: 616 fh.write(self.getCmds(subst,j)+"\n") 617 618 if loopVars: 619 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 620 fh.write(loopFoot+"\n") 621 622 fh.write(footer+"\n") 623 624 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 625 #if '10_9' in testname: sys.exit() 626 return 627 628 def genScriptsAndInfo(self,exfile,root,srcDict): 629 """ 630 Generate scripts from the source file, determine if built, etc. 631 For every test in the exfile with info in the srcDict: 632 1. Determine if it needs to be run for this arch 633 2. Generate the script 634 3. Generate the data needed to write out the makefile in a 635 convenient way 636 All tests are *always* run, but some may be SKIP'd per the TAP standard 637 """ 638 debug=False 639 rpath=self.srcrelpath(root) 640 execname=self.getExecname(exfile,rpath) 641 isBuilt=self._isBuilt(exfile,srcDict) 642 for test in srcDict: 643 if test in self.buildkeys: continue 644 if debug: print(nameSpace(exfile,root), test) 645 srcDict[test]['execname']=execname # Convenience in generating scripts 646 isRun=self._isRun(srcDict[test]) 647 self.genRunScript(test,root,isRun,srcDict) 648 srcDict[test]['isrun']=isRun 649 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 650 651 # This adds to datastructure for building deps 652 if isBuilt: self.addToSources(exfile,rpath,srcDict) 653 return 654 655 def _isBuilt(self,exfile,srcDict): 656 """ 657 Determine if this file should be built. 658 """ 659 # Get the language based on file extension 660 srcDict['SKIP'] = [] 661 lang=self.getLanguage(exfile) 662 if (lang=="F" or lang=="F90"): 663 if not self.have_fortran: 664 srcDict["SKIP"].append("Fortran required for this test") 665 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 666 srcDict["SKIP"].append("Fortran f90freeform required for this test") 667 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 668 srcDict["SKIP"].append("CUDA required for this test") 669 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 670 srcDict["SKIP"].append("HIP required for this test") 671 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 672 srcDict["SKIP"].append("SYCL required for this test") 673 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 674 srcDict["SKIP"].append("KOKKOS required for this test") 675 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 676 srcDict["SKIP"].append("RAJA required for this test") 677 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 678 srcDict["SKIP"].append("C++ required for this test") 679 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 680 srcDict["SKIP"].append("C++ required for this test") 681 682 # Deprecated source files 683 if srcDict.get("TODO"): 684 return False 685 686 # isRun can work with srcDict to handle the requires 687 if "requires" in srcDict: 688 if srcDict["requires"]: 689 return self._isRun(srcDict) 690 691 return srcDict['SKIP'] == [] 692 693 694 def _isRun(self,testDict, debug=False): 695 """ 696 Based on the requirements listed in the src file and the petscconf.h 697 info, determine whether this test should be run or not. 698 """ 699 indent=" " 700 701 if 'SKIP' not in testDict: 702 testDict['SKIP'] = [] 703 # MPI requirements 704 if 'MPI_IS_MPIUNI' in self.conf: 705 if testDict.get('nsize', '1') != '1': 706 testDict['SKIP'].append("Parallel test with serial build") 707 708 # The requirements for the test are the sum of all the run subtests 709 if 'subtests' in testDict: 710 if 'requires' not in testDict: testDict['requires']="" 711 for stest in testDict['subtests']: 712 if 'requires' in testDict[stest]: 713 testDict['requires']+=" "+testDict[stest]['requires'] 714 if testDict[stest].get('nsize', '1') != '1': 715 testDict['SKIP'].append("Parallel test with serial build") 716 break 717 718 # Now go through all requirements 719 if 'requires' in testDict: 720 for requirement in testDict['requires'].split(): 721 requirement=requirement.strip() 722 if not requirement: continue 723 if debug: print(indent+"Requirement: ", requirement) 724 isNull=False 725 if requirement.startswith("!"): 726 requirement=requirement[1:]; isNull=True 727 # Precision requirement for reals 728 if requirement in self.precision_types: 729 if self.conf['PETSC_PRECISION']==requirement: 730 if isNull: 731 testDict['SKIP'].append("not "+requirement+" required") 732 continue 733 continue # Success 734 elif not isNull: 735 testDict['SKIP'].append(requirement+" required") 736 continue 737 # Precision requirement for ints 738 if requirement in self.integer_types: 739 if requirement=="int32": 740 if self.conf['PETSC_SIZEOF_INT']==4: 741 if isNull: 742 testDict['SKIP'].append("not int32 required") 743 continue 744 continue # Success 745 elif not isNull: 746 testDict['SKIP'].append("int32 required") 747 continue 748 if requirement=="int64": 749 if self.conf['PETSC_SIZEOF_INT']==8: 750 if isNull: 751 testDict['SKIP'].append("NOT int64 required") 752 continue 753 continue # Success 754 elif not isNull: 755 testDict['SKIP'].append("int64 required") 756 continue 757 if requirement.startswith("long"): 758 reqsize = int(requirement[4:])//8 759 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 760 if longsize==reqsize: 761 if isNull: 762 testDict['SKIP'].append("not %s required" % requirement) 763 continue 764 continue # Success 765 elif not isNull: 766 testDict['SKIP'].append("%s required" % requirement) 767 continue 768 # Datafilespath 769 if requirement=="datafilespath" and not isNull: 770 testDict['SKIP'].append("Requires DATAFILESPATH") 771 continue 772 # Defines -- not sure I have comments matching 773 if "defined(" in requirement.lower(): 774 reqdef=requirement.split("(")[1].split(")")[0] 775 if reqdef in self.conf: 776 if isNull: 777 testDict['SKIP'].append("Null requirement not met: "+requirement) 778 continue 779 continue # Success 780 elif not isNull: 781 testDict['SKIP'].append("Required: "+requirement) 782 continue 783 784 # Rest should be packages that we can just get from conf 785 if requirement in ["complex","debug"]: 786 petscconfvar="PETSC_USE_"+requirement.upper() 787 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 788 else: 789 petscconfvar="PETSC_HAVE_"+requirement.upper() 790 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 791 petsccv = self.conf.get(petscconfvar) 792 pkgcv = self.conf.get(pkgconfvar) 793 794 if petsccv or pkgcv: 795 if isNull: 796 if petsccv: 797 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 798 continue 799 else: 800 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 801 continue 802 continue # Success 803 elif not isNull: 804 if not petsccv and not pkgcv: 805 if debug: print("requirement not found: ", requirement) 806 if self.pkg_name == 'petsc': 807 testDict['SKIP'].append(petscconfvar+" requirement not met") 808 else: 809 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 810 continue 811 return testDict['SKIP'] == [] 812 813 def checkOutput(self,exfile,root,srcDict): 814 """ 815 Check and make sure the output files are in the output directory 816 """ 817 debug=False 818 rpath=self.srcrelpath(root) 819 for test in srcDict: 820 if test in self.buildkeys: continue 821 if debug: print(rpath, exfile, test) 822 if 'output_file' in srcDict[test]: 823 output_file=srcDict[test]['output_file'] 824 else: 825 defroot = testparse.getDefaultOutputFileRoot(test) 826 if 'TODO' in srcDict[test]: continue 827 output_file="output/"+defroot+".out" 828 829 fullout=os.path.join(root,output_file) 830 if debug: print("---> ",fullout) 831 if not os.path.exists(fullout): 832 self.missing_files.append(fullout) 833 834 return 835 836 def genPetscTests_summarize(self,dataDict): 837 """ 838 Required method to state what happened 839 """ 840 if not self.summarize: return 841 indent=" " 842 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 843 with open(fhname, "w") as fh: 844 for root in dataDict: 845 relroot=self.srcrelpath(root) 846 pkg=relroot.split("/")[1] 847 fh.write(relroot+"\n") 848 allSrcs=[] 849 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 850 for exfile in dataDict[root]: 851 # Basic information 852 rfile=os.path.join(relroot,exfile) 853 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 854 fh.write(indent+exfile+indent*4+builtStatus+"\n") 855 for test in dataDict[root][exfile]: 856 if test in self.buildkeys: continue 857 line=indent*2+test 858 fh.write(line+"\n") 859 # Looks nice to have the keys in order 860 #for key in dataDict[root][exfile][test]: 861 for key in "isrun abstracted nsize args requires script".split(): 862 if key not in dataDict[root][exfile][test]: continue 863 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 864 fh.write(line+"\n") 865 fh.write("\n") 866 fh.write("\n") 867 fh.write("\n") 868 return 869 870 def genPetscTests(self,root,dirs,files,dataDict): 871 """ 872 Go through and parse the source files in the directory to generate 873 the examples based on the metadata contained in the source files 874 """ 875 debug=False 876 # Use examplesAnalyze to get what the makefles think are sources 877 #self.examplesAnalyze(root,dirs,files,anlzDict) 878 879 dataDict[root]={} 880 881 for exfile in files: 882 #TST: Until we replace files, still leaving the orginals as is 883 #if not exfile.startswith("new_"+"ex"): continue 884 #if not exfile.startswith("ex"): continue 885 886 # Ignore emacs and other temporary files 887 if exfile.startswith("."): continue 888 if exfile.startswith("#"): continue 889 if exfile.endswith("~"): continue 890 # Only parse source files 891 ext=getlangext(exfile).lstrip('.').replace('.','_') 892 if ext not in LANGS: continue 893 894 # Convenience 895 fullex=os.path.join(root,exfile) 896 if self.verbose: print(' --> '+fullex) 897 dataDict[root].update(testparse.parseTestFile(fullex,0)) 898 if exfile in dataDict[root]: 899 if not self.check_output: 900 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 901 else: 902 self.checkOutput(exfile,root,dataDict[root][exfile]) 903 904 return 905 906 def walktree(self,top): 907 """ 908 Walk a directory tree, starting from 'top' 909 """ 910 if self.check_output: 911 print("Checking for missing output files") 912 self.missing_files=[] 913 914 # Goal of action is to fill this dictionary 915 dataDict={} 916 for root, dirs, files in os.walk(top, topdown=True): 917 dirs.sort() 918 files.sort() 919 if "/tests" not in root and "/tutorials" not in root: continue 920 if "dSYM" in root: continue 921 if "tutorials"+os.sep+"build" in root: continue 922 if os.path.basename(root.rstrip("/")) == 'output': continue 923 if self.verbose: print(root) 924 self.genPetscTests(root,dirs,files,dataDict) 925 926 # If checking output, report results 927 if self.check_output: 928 if self.missing_files: 929 for file in set(self.missing_files): # set uniqifies 930 print(file) 931 sys.exit(1) 932 933 # Now summarize this dictionary 934 if self.verbose: self.genPetscTests_summarize(dataDict) 935 return dataDict 936 937 def gen_gnumake(self, fd): 938 """ 939 Overwrite of the method in the base PETSc class 940 """ 941 def write(stem, srcs): 942 for lang in LANGS: 943 if srcs[lang]['srcs']: 944 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 945 for pkg in self.pkg_pkgs: 946 srcs = self.gen_pkg(pkg) 947 write('testsrcs-' + pkg, srcs) 948 # Handle dependencies 949 for lang in LANGS: 950 for exfile in srcs[lang]['srcs']: 951 if exfile in srcs[lang]: 952 ex='$(TESTDIR)/'+getlangsplit(exfile) 953 exfo=ex+'.o' 954 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 955 if deps: 956 # The executable literally depends on the object file because it is linked 957 fd.write(ex +": " + " ".join(deps) +'\n') 958 # The object file containing 'main' does not normally depend on other object 959 # files, but it does when it includes their modules. This dependency is 960 # overly blunt and could be reduced to only depend on object files for 961 # modules that are used, like "*f90aux.o". 962 fd.write(exfo +": " + " ".join(deps) +'\n') 963 964 return self.gendeps 965 966 def gen_pkg(self, pkg): 967 """ 968 Overwrite of the method in the base PETSc class 969 """ 970 return self.sources[pkg] 971 972 def write_gnumake(self, dataDict, output=None): 973 """ 974 Write out something similar to files from gmakegen.py 975 976 Test depends on script which also depends on source 977 file, but since I don't have a good way generating 978 acting on a single file (oops) just depend on 979 executable which in turn will depend on src file 980 """ 981 # Different options for how to set up the targets 982 compileExecsFirst=False 983 984 # Open file 985 with open(output, 'w') as fd: 986 # Write out the sources 987 gendeps = self.gen_gnumake(fd) 988 989 # Write out the tests and execname targets 990 fd.write("\n#Tests and executables\n") # Delimiter 991 992 for pkg in self.pkg_pkgs: 993 # These grab the ones that are built 994 for lang in LANGS: 995 testdeps=[] 996 for ftest in self.tests[pkg][lang]: 997 test=os.path.basename(ftest) 998 basedir=os.path.dirname(ftest) 999 testdeps.append(nameSpace(test,basedir)) 1000 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1001 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1002 1003 # test targets 1004 for ftest in self.tests[pkg][lang]: 1005 test=os.path.basename(ftest) 1006 basedir=os.path.dirname(ftest) 1007 testdir="${TESTDIR}/"+basedir+"/" 1008 nmtest=nameSpace(test,basedir) 1009 rundir=os.path.join(testdir,test) 1010 script=test+".sh" 1011 1012 # Deps 1013 exfile=self.tests[pkg][lang][ftest]['exfile'] 1014 fullex=os.path.join(self.srcdir,exfile) 1015 localexec=self.tests[pkg][lang][ftest]['exec'] 1016 execname=os.path.join(testdir,localexec) 1017 fullscript=os.path.join(testdir,script) 1018 tmpfile=os.path.join(testdir,test,test+".tmp") 1019 1020 # *.counts depends on the script and either executable (will 1021 # be run) or the example source file (SKIP or TODO) 1022 fd.write('%s.counts : %s %s' 1023 % (os.path.join('$(TESTDIR)/counts', nmtest), 1024 fullscript, 1025 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1026 ) 1027 if exfile in self.sources[pkg][lang]: 1028 for dep in self.sources[pkg][lang][exfile]: 1029 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1030 fd.write('\n') 1031 1032 # Now write the args: 1033 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1034 1035 return 1036 1037 def write_db(self, dataDict, testdir): 1038 """ 1039 Write out the dataDict into a pickle file 1040 """ 1041 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1042 pickle.dump(dataDict,fd) 1043 return 1044 1045def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1046 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1047 srcdir=None, testdir=None, check=False): 1048 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1049 testdir=os.path.normpath(testdir) 1050 if petsc_arch: 1051 petsc_arch=petsc_arch.rstrip(os.path.sep) 1052 if len(petsc_arch.split(os.path.sep))>1: 1053 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1054 output = os.path.join(testdir, 'testfiles') 1055 1056 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1057 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1058 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1059 testdir=testdir,check=check) 1060 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1061 if not pEx.check_output: 1062 pEx.write_gnumake(dataDict, output) 1063 pEx.write_db(dataDict, testdir) 1064 1065if __name__ == '__main__': 1066 import optparse 1067 parser = optparse.OptionParser() 1068 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1069 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1070 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1071 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1072 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1073 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1074 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1075 help='Check whether output files are in output director') 1076 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1077 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1078 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1079 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1080 1081 opts, extra_args = parser.parse_args() 1082 if extra_args: 1083 import sys 1084 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1085 exit(1) 1086 if opts.testdir is None: 1087 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1088 1089 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1090 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1091 verbose=opts.verbose, 1092 single_ex=opts.single_executable, srcdir=opts.srcdir, 1093 testdir=opts.testdir, check=opts.check_output) 1094