1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9import shlex 10sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 11from collections import defaultdict 12from gmakegen import * 13 14import inspect 15thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 16sys.path.insert(0,thisscriptdir) 17import testparse 18import example_template 19 20 21""" 22 23There are 2 modes of running tests: Normal builds and run from prefix of 24install. They affect where to find things: 25 26 27Case 1. Normal builds: 28 29 +---------------------+----------------------------------+ 30 | PETSC_DIR | <git dir> | 31 +---------------------+----------------------------------+ 32 | PETSC_ARCH | arch-foo | 33 +---------------------+----------------------------------+ 34 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 35 +---------------------+----------------------------------+ 36 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 37 +---------------------+----------------------------------+ 38 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 39 +---------------------+----------------------------------+ 40 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 41 +---------------------+----------------------------------+ 42 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 43 +---------------------+----------------------------------+ 44 45 46Case 2. From install dir: 47 48 +---------------------+-------------------------------------------------------+ 49 | PETSC_DIR | <prefix dir> | 50 +---------------------+-------------------------------------------------------+ 51 | PETSC_ARCH | '' | 52 +---------------------+-------------------------------------------------------+ 53 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 54 +---------------------+-------------------------------------------------------+ 55 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 56 +---------------------+-------------------------------------------------------+ 57 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 58 +---------------------+-------------------------------------------------------+ 59 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 60 +---------------------+-------------------------------------------------------+ 61 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 62 +---------------------+-------------------------------------------------------+ 63 64""" 65 66def install_files(source, destdir): 67 """Install file or directory 'source' to 'destdir'. Does not preserve 68 mode (permissions). 69 """ 70 if not os.path.isdir(destdir): 71 os.makedirs(destdir) 72 if os.path.isdir(source): 73 for name in os.listdir(source): 74 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 75 else: 76 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 77 78def nameSpace(srcfile,srcdir): 79 """ 80 Because the scripts have a non-unique naming, the pretty-printing 81 needs to convey the srcdir and srcfile. There are two ways of doing this. 82 """ 83 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 84 prefix=srcdir.replace("/","_")+"-" 85 nameString=prefix+srcfile 86 return nameString 87 88class generateExamples(Petsc): 89 """ 90 gmakegen.py has basic structure for finding the files, writing out 91 the dependencies, etc. 92 """ 93 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 94 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 95 96 self.single_ex=single_ex 97 self.srcdir=srcdir 98 self.check_output=check 99 100 # Set locations to handle movement 101 self.inInstallDir=self.getInInstallDir(thisscriptdir) 102 103 # Special configuration for CI testing 104 if self.petsc_arch.find('valgrind') >= 0: 105 self.conf['PETSCTEST_VALGRIND']=1 106 107 if self.inInstallDir: 108 # Case 2 discussed above 109 # set PETSC_ARCH to install directory to allow script to work in both 110 dirlist=thisscriptdir.split(os.path.sep) 111 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 112 self.arch_dir=installdir 113 if self.srcdir is None: 114 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 115 else: 116 if petsc_arch == '': 117 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 118 # Case 1 discussed above 119 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 120 if self.srcdir is None: 121 self.srcdir=os.path.join(self.petsc_dir,'src') 122 123 self.testroot_dir=os.path.abspath(testdir) 124 125 self.verbose=verbose 126 # Whether to write out a useful debugging 127 self.summarize=True if verbose else False 128 129 # For help in setting the requirements 130 self.precision_types="__fp16 single double __float128".split() 131 self.integer_types="int32 int64 long32 long64".split() 132 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 133 134 # Things that are not test 135 self.buildkeys=testparse.buildkeys 136 137 # Adding a dictionary for storing sources, objects, and tests 138 # to make building the dependency tree easier 139 self.sources={} 140 self.objects={} 141 self.tests={} 142 for pkg in self.pkg_pkgs: 143 self.sources[pkg]={} 144 self.objects[pkg]=[] 145 self.tests[pkg]={} 146 for lang in LANGS: 147 self.sources[pkg][lang]={} 148 self.sources[pkg][lang]['srcs']=[] 149 self.tests[pkg][lang]={} 150 151 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 152 153 self.indent=" " 154 if self.verbose: print('Finishing the constructor') 155 return 156 157 def srcrelpath(self,rdir): 158 """ 159 Get relative path to source directory 160 """ 161 return os.path.relpath(rdir,self.srcdir) 162 163 def getInInstallDir(self,thisscriptdir): 164 """ 165 When petsc is installed then this file in installed in: 166 <PREFIX>/share/petsc/examples/config/gmakegentest.py 167 otherwise the path is: 168 <PETSC_DIR>/config/gmakegentest.py 169 We use this difference to determine if we are in installdir 170 """ 171 dirlist=thisscriptdir.split(os.path.sep) 172 if len(dirlist)>4: 173 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 174 if lastfour==os.path.join('share','petsc','examples','config'): 175 return True 176 else: 177 return False 178 else: 179 return False 180 181 def getLanguage(self,srcfile): 182 """ 183 Based on the source, determine associated language as found in gmakegen.LANGS 184 Can we just return srcext[1:\] now? 185 """ 186 langReq=None 187 srcext = getlangext(srcfile) 188 if srcext in ".F90".split(): langReq="F90" 189 if srcext in ".F".split(): langReq="F" 190 if srcext in ".cxx".split(): langReq="cxx" 191 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 192 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 193 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 194 if srcext in ".cpp".split(): langReq="cpp" 195 if srcext == ".cu": langReq="cu" 196 if srcext == ".c": langReq="c" 197 #if not langReq: print("ERROR: ", srcext, srcfile) 198 return langReq 199 200 def _getAltList(self,output_file,srcdir): 201 ''' Calculate AltList based on output file-- see 202 src/snes/tutorials/output/ex22*.out 203 ''' 204 altlist=[output_file] 205 basefile = getlangsplit(output_file) 206 for i in range(1,9): 207 altroot=basefile+"_alt" 208 if i > 1: altroot=altroot+"_"+str(i) 209 af=altroot+".out" 210 srcaf=os.path.join(srcdir,af) 211 fullaf=os.path.join(self.petsc_dir,srcaf) 212 if os.path.isfile(fullaf): altlist.append(srcaf) 213 214 return altlist 215 216 217 def _getLoopVars(self,inDict,testname, isSubtest=False): 218 """ 219 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 220 Return: 221 inDict['args']: -ksp_monitor 222 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 223 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 224 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 225 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 226 subst should be passed in instead of inDict 227 """ 228 loopVars={}; newargs=[] 229 lsuffix='+' 230 argregex = re.compile(' (?=-[a-zA-Z])') 231 from testparse import parseLoopArgs 232 for key in inDict: 233 if key in ('SKIP', 'regexes'): 234 continue 235 akey=('subargs' if key=='args' else key) # what to assign 236 if akey not in inDict: inDict[akey]='' 237 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 238 # Always generate a loop over nsize, even if there is only one value 239 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 240 keystr = str(inDict[key]) 241 varlist = [] 242 for varset in argregex.split(keystr): 243 if not varset.strip(): continue 244 if '{{' in varset: 245 keyvar,lvars,ftype=parseLoopArgs(varset) 246 if akey not in loopVars: loopVars[akey]={} 247 varlist.append(keyvar) 248 loopVars[akey][keyvar]=[keyvar,lvars] 249 if akey=='nsize': 250 if len(lvars.split()) > 1: 251 lsuffix += akey +'-${i' + keyvar + '}' 252 else: 253 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 254 lsuffix+=keyvar+'-${i' + keyvar + '}_' 255 else: 256 if key=='args': 257 newargs.append(varset.strip()) 258 if varlist: 259 loopVars[akey]['varlist']=varlist 260 261 # For subtests, args are always substituted in (not top level) 262 if isSubtest: 263 inDict['subargs'] += " "+" ".join(newargs) 264 inDict['args']='' 265 if 'label_suffix' in inDict: 266 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 267 else: 268 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 269 else: 270 if loopVars: 271 inDict['args'] = ' '.join(newargs) 272 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 273 return loopVars 274 275 def getArgLabel(self,testDict): 276 """ 277 In all of the arguments in the test dictionary, create a simple 278 string for searching within the makefile system. For simplicity in 279 search, remove "-", for strings, etc. 280 Also, concatenate the arg commands 281 For now, ignore nsize -- seems hard to search for anyway 282 """ 283 # Collect all of the args associated with a test 284 argStr=("" if 'args' not in testDict else testDict['args']) 285 if 'subtests' in testDict: 286 for stest in testDict["subtests"]: 287 sd=testDict[stest] 288 argStr=argStr+("" if 'args' not in sd else sd['args']) 289 290 # Now go through and cleanup 291 argStr=re.sub('{{(.*?)}}',"",argStr) 292 argStr=re.sub('-'," ",argStr) 293 for digit in string.digits: argStr=re.sub(digit," ",argStr) 294 argStr=re.sub("\.","",argStr) 295 argStr=re.sub(",","",argStr) 296 argStr=re.sub('\+',' ',argStr) 297 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 298 return argStr.strip() 299 300 def addToSources(self,exfile,rpath,srcDict): 301 """ 302 Put into data structure that allows easy generation of makefile 303 """ 304 pkg=rpath.split(os.path.sep)[0] 305 relpfile=os.path.join(rpath,exfile) 306 lang=self.getLanguage(exfile) 307 if not lang: return 308 if pkg not in self.sources: return 309 self.sources[pkg][lang]['srcs'].append(relpfile) 310 self.sources[pkg][lang][relpfile] = [] 311 if 'depends' in srcDict: 312 depSrcList=srcDict['depends'].split() 313 for depSrc in depSrcList: 314 depObj = getlangsplit(depSrc)+'.o' 315 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 316 317 # In gmakefile, ${TESTDIR} var specifies the object compilation 318 testsdir=rpath+"/" 319 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 320 self.objects[pkg].append(objfile) 321 return 322 323 def addToTests(self,test,rpath,exfile,execname,testDict): 324 """ 325 Put into data structure that allows easy generation of makefile 326 Organized by languages to allow testing of languages 327 """ 328 pkg=rpath.split("/")[0] 329 nmtest=os.path.join(rpath,test) 330 lang=self.getLanguage(exfile) 331 if not lang: return 332 if pkg not in self.tests: return 333 self.tests[pkg][lang][nmtest]={} 334 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 335 self.tests[pkg][lang][nmtest]['exec']=execname 336 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 337 return 338 339 def getExecname(self,exfile,rpath): 340 """ 341 Generate bash script using template found next to this file. 342 This file is read in at constructor time to avoid file I/O 343 """ 344 if self.single_ex: 345 execname=rpath.split("/")[1]+"-ex" 346 else: 347 execname=getlangsplit(exfile) 348 return execname 349 350 def getSubstVars(self,testDict,rpath,testname): 351 """ 352 Create a dictionary with all of the variables that get substituted 353 into the template commands found in example_template.py 354 """ 355 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 356 if 'nsize' not in testDict: testDict['nsize'] = '1' 357 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 358 subst = {key : testDict.get(key, '') for key in testparse.acceptedkeys if key != 'test'} 359 360 # Now do other variables 361 subst['env'] = '\n'.join('export '+cmd for cmd in shlex.split(subst['env'])) 362 subst['execname']=testDict['execname'] 363 subst['error']='' 364 if 'filter' in testDict: 365 if testDict['filter'].startswith("Error:"): 366 subst['error']="Error" 367 subst['filter']=testDict['filter'].lstrip("Error:") 368 else: 369 subst['filter']=testDict['filter'] 370 371 # Others 372 subst['subargs']='' # Default. For variables override 373 subst['srcdir']=os.path.join(self.srcdir, rpath) 374 subst['label_suffix']='' 375 subst['comments']="\n#".join(subst['comments'].split("\n")) 376 if subst['comments']: subst['comments']="#"+subst['comments'] 377 subst['exec']="../"+subst['execname'] 378 subst['testroot']=self.testroot_dir 379 subst['testname']=testname 380 dp = self.conf.get('DATAFILESPATH','') 381 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 382 383 # This is used to label some matrices 384 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 385 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 386 387 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 388 389 #Conf vars 390 if self.petsc_arch.find('valgrind')>=0: 391 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 392 else: 393 subst['mpiexec']=self.conf['MPIEXEC'] 394 subst['pkg_name']=self.pkg_name 395 subst['pkg_dir']=self.pkg_dir 396 subst['pkg_arch']=self.petsc_arch 397 subst['CONFIG_DIR']=thisscriptdir 398 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 399 subst['diff']=self.conf['DIFF'] 400 subst['rm']=self.conf['RM'] 401 subst['grep']=self.conf['GREP'] 402 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 403 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 404 405 # Output file is special because of subtests override 406 defroot = testparse.getDefaultOutputFileRoot(testname) 407 if 'output_file' not in testDict: 408 subst['output_file']="output/"+defroot+".out" 409 subst['redirect_file']=defroot+".tmp" 410 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 411 412 # Add in the full path here. 413 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 414 415 subst['regexes']={} 416 for subkey in subst: 417 if subkey=='regexes': continue 418 if not isinstance(subst[subkey],str): continue 419 patt="@"+subkey.upper()+"@" 420 subst['regexes'][subkey]=re.compile(patt) 421 422 return subst 423 424 def _substVars(self,subst,origStr): 425 """ 426 Substitute variables 427 """ 428 Str=origStr 429 for subkey, subvalue in subst.items(): 430 if subkey=='regexes': continue 431 if not isinstance(subvalue,str): continue 432 if subkey.upper() not in Str: continue 433 Str=subst['regexes'][subkey].sub(lambda x: subvalue,Str) 434 return Str 435 436 def getCmds(self,subst,i, debug=False): 437 """ 438 Generate bash script using template found next to this file. 439 This file is read in at constructor time to avoid file I/O 440 """ 441 nindnt=i # the start and has to be consistent with below 442 cmdindnt=self.indent*nindnt 443 cmdLines="" 444 445 # MPI is the default -- but we have a few odd commands 446 if not subst['command']: 447 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 448 else: 449 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 450 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 451 452 cmdLines+=cmdindnt+'if test $res = 0; then\n' 453 diffindnt=self.indent*(nindnt+1) 454 455 # Do some checks on existence of output_file and alt files 456 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 457 if not subst['TODO']: 458 print("Warning: "+subst['output_file']+" not found.") 459 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 460 461 # altlist always has output_file 462 if len(altlist)==1: 463 cmd=diffindnt+self._substVars(subst,example_template.difftest) 464 else: 465 if debug: print("Found alt files: ",altlist) 466 # Have to do it by hand a bit because of variable number of alt files 467 rf=subst['redirect_file'] 468 cmd=diffindnt+example_template.difftest.split('@')[0] 469 for i in range(len(altlist)): 470 af=altlist[i] 471 cmd+=af+' '+rf 472 if i!=len(altlist)-1: 473 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 474 cmd+=' || ${diff_exe} ' 475 else: 476 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 477 cmd+=subst['label_suffix']+' ""' # Quotes are painful 478 cmdLines+=cmd+"\n" 479 cmdLines+=cmdindnt+'else\n' 480 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 481 cmdLines+=cmdindnt+'fi\n' 482 return cmdLines 483 484 def _writeTodoSkip(self,fh,tors,reasons,footer): 485 """ 486 Write out the TODO and SKIP lines in the file 487 The TODO or SKIP variable, tors, should be lower case 488 """ 489 TORS=tors.upper() 490 template=eval("example_template."+tors+"line") 491 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 492 tab = '' 493 if reasons: 494 fh.write('if ! $force; then\n') 495 tab = tab + ' ' 496 if reasons == ["Requires DATAFILESPATH"]: 497 # The only reason not to run is DATAFILESPATH, which we check at run-time 498 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 499 tab = tab + ' ' 500 if reasons: 501 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 502 fh.write(tab+footer+"\n") 503 fh.write(tab+"exit\n") 504 if reasons == ["Requires DATAFILESPATH"]: 505 fh.write(' fi\n') 506 if reasons: 507 fh.write('fi\n') 508 fh.write('\n\n') 509 return 510 511 def getLoopVarsHead(self,loopVars,i,usedVars={}): 512 """ 513 Generate a nicely indented string with the format loops 514 Here is what the data structure looks like 515 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 516 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 517 loopVars['subargs']['pc_type']=["j","cholesky sor"] 518 """ 519 outstr=''; indnt=self.indent 520 521 for key in loopVars: 522 for var in loopVars[key]['varlist']: 523 varval=loopVars[key][var] 524 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 525 outstr += "\n\n" 526 527 for key in loopVars: 528 for var in loopVars[key]['varlist']: 529 varval=loopVars[key][var] 530 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 531 i = i + 1 532 return (outstr,i) 533 534 def getLoopVarsFoot(self,loopVars,i): 535 outstr=''; indnt=self.indent 536 for key in loopVars: 537 for var in loopVars[key]['varlist']: 538 i = i - 1 539 outstr += indnt * i + "done\n" 540 return (outstr,i) 541 542 def genRunScript(self,testname,root,isRun,srcDict): 543 """ 544 Generate bash script using template found next to this file. 545 This file is read in at constructor time to avoid file I/O 546 """ 547 def opener(path,flags,*args,**kwargs): 548 kwargs.setdefault('mode',0o755) 549 return os.open(path,flags,*args,**kwargs) 550 551 # runscript_dir directory has to be consistent with gmakefile 552 testDict=srcDict[testname] 553 rpath=self.srcrelpath(root) 554 runscript_dir=os.path.join(self.testroot_dir,rpath) 555 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 556 with open(os.path.join(runscript_dir,testname+".sh"),"w",opener=opener) as fh: 557 558 # Get variables to go into shell scripts. last time testDict used 559 subst=self.getSubstVars(testDict,rpath,testname) 560 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 561 if 'subtests' in testDict: 562 # The subtests inherit inDict, so we don't need top-level loops. 563 loopVars = {} 564 565 #Handle runfiles 566 for lfile in subst.get('localrunfiles','').split(): 567 install_files(os.path.join(root, lfile), 568 os.path.join(runscript_dir, os.path.dirname(lfile))) 569 # Check subtests for local runfiles 570 for stest in subst.get("subtests",[]): 571 for lfile in testDict[stest].get('localrunfiles','').split(): 572 install_files(os.path.join(root, lfile), 573 os.path.join(runscript_dir, os.path.dirname(lfile))) 574 575 # Now substitute the key variables into the header and footer 576 header=self._substVars(subst,example_template.header) 577 # The header is done twice to enable @...@ in header 578 header=self._substVars(subst,header) 579 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 580 581 # Start writing the file 582 fh.write(header+"\n") 583 584 # If there is a TODO or a SKIP then we do it before writing out the 585 # rest of the command (which is useful for working on the test) 586 # SKIP and TODO can be for the source file or for the runs 587 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 588 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 589 590 j=0 # for indentation 591 592 if loopVars: 593 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 594 if (loopHead): fh.write(loopHead+"\n") 595 596 # Subtests are special 597 allLoopVars=list(loopVars.keys()) 598 if 'subtests' in testDict: 599 substP=subst # Subtests can inherit args but be careful 600 k=0 # for label suffixes 601 for stest in testDict["subtests"]: 602 subst=substP.copy() 603 subst.update(testDict[stest]) 604 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 605 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 606 if sLoopVars: 607 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 608 allLoopVars+=list(sLoopVars.keys()) 609 fh.write(sLoopHead+"\n") 610 fh.write(self.getCmds(subst,j)+"\n") 611 if sLoopVars: 612 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 613 fh.write(sLoopFoot+"\n") 614 else: 615 fh.write(self.getCmds(subst,j)+"\n") 616 617 if loopVars: 618 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 619 fh.write(loopFoot+"\n") 620 621 fh.write(footer+"\n") 622 return 623 624 def genScriptsAndInfo(self,exfile,root,srcDict): 625 """ 626 Generate scripts from the source file, determine if built, etc. 627 For every test in the exfile with info in the srcDict: 628 1. Determine if it needs to be run for this arch 629 2. Generate the script 630 3. Generate the data needed to write out the makefile in a 631 convenient way 632 All tests are *always* run, but some may be SKIP'd per the TAP standard 633 """ 634 debug=False 635 rpath=self.srcrelpath(root) 636 execname=self.getExecname(exfile,rpath) 637 isBuilt=self._isBuilt(exfile,srcDict) 638 for test in srcDict: 639 if test in self.buildkeys: continue 640 if debug: print(nameSpace(exfile,root), test) 641 srcDict[test]['execname']=execname # Convenience in generating scripts 642 isRun=self._isRun(srcDict[test]) 643 self.genRunScript(test,root,isRun,srcDict) 644 srcDict[test]['isrun']=isRun 645 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 646 647 # This adds to datastructure for building deps 648 if isBuilt: self.addToSources(exfile,rpath,srcDict) 649 return 650 651 def _isBuilt(self,exfile,srcDict): 652 """ 653 Determine if this file should be built. 654 """ 655 # Get the language based on file extension 656 srcDict['SKIP'] = [] 657 lang=self.getLanguage(exfile) 658 if (lang=="F" or lang=="F90"): 659 if not self.have_fortran: 660 srcDict["SKIP"].append("Fortran required for this test") 661 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 662 srcDict["SKIP"].append("Fortran f90freeform required for this test") 663 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 664 srcDict["SKIP"].append("CUDA required for this test") 665 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 666 srcDict["SKIP"].append("HIP required for this test") 667 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 668 srcDict["SKIP"].append("SYCL required for this test") 669 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 670 srcDict["SKIP"].append("KOKKOS required for this test") 671 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 672 srcDict["SKIP"].append("RAJA required for this test") 673 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 674 srcDict["SKIP"].append("C++ required for this test") 675 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 676 srcDict["SKIP"].append("C++ required for this test") 677 678 # Deprecated source files 679 if srcDict.get("TODO"): 680 return False 681 682 # isRun can work with srcDict to handle the requires 683 if "requires" in srcDict: 684 if srcDict["requires"]: 685 return self._isRun(srcDict) 686 687 return srcDict['SKIP'] == [] 688 689 690 def _isRun(self,testDict, debug=False): 691 """ 692 Based on the requirements listed in the src file and the petscconf.h 693 info, determine whether this test should be run or not. 694 """ 695 indent=" " 696 697 if 'SKIP' not in testDict: 698 testDict['SKIP'] = [] 699 # MPI requirements 700 if 'MPI_IS_MPIUNI' in self.conf: 701 if testDict.get('nsize', '1') != '1': 702 testDict['SKIP'].append("Parallel test with serial build") 703 704 # The requirements for the test are the sum of all the run subtests 705 if 'subtests' in testDict: 706 if 'requires' not in testDict: testDict['requires']="" 707 for stest in testDict['subtests']: 708 if 'requires' in testDict[stest]: 709 testDict['requires']+=" "+testDict[stest]['requires'] 710 if testDict[stest].get('nsize', '1') != '1': 711 testDict['SKIP'].append("Parallel test with serial build") 712 break 713 714 # Now go through all requirements 715 if 'requires' in testDict: 716 for requirement in testDict['requires'].split(): 717 requirement=requirement.strip() 718 if not requirement: continue 719 if debug: print(indent+"Requirement: ", requirement) 720 isNull=False 721 if requirement.startswith("!"): 722 requirement=requirement[1:]; isNull=True 723 # Precision requirement for reals 724 if requirement in self.precision_types: 725 if self.conf['PETSC_PRECISION']==requirement: 726 if isNull: 727 testDict['SKIP'].append("not "+requirement+" required") 728 continue 729 continue # Success 730 elif not isNull: 731 testDict['SKIP'].append(requirement+" required") 732 continue 733 # Precision requirement for ints 734 if requirement in self.integer_types: 735 if requirement=="int32": 736 if self.conf['PETSC_SIZEOF_INT']==4: 737 if isNull: 738 testDict['SKIP'].append("not int32 required") 739 continue 740 continue # Success 741 elif not isNull: 742 testDict['SKIP'].append("int32 required") 743 continue 744 if requirement=="int64": 745 if self.conf['PETSC_SIZEOF_INT']==8: 746 if isNull: 747 testDict['SKIP'].append("NOT int64 required") 748 continue 749 continue # Success 750 elif not isNull: 751 testDict['SKIP'].append("int64 required") 752 continue 753 if requirement.startswith("long"): 754 reqsize = int(requirement[4:])//8 755 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 756 if longsize==reqsize: 757 if isNull: 758 testDict['SKIP'].append("not %s required" % requirement) 759 continue 760 continue # Success 761 elif not isNull: 762 testDict['SKIP'].append("%s required" % requirement) 763 continue 764 # Datafilespath 765 if requirement=="datafilespath" and not isNull: 766 testDict['SKIP'].append("Requires DATAFILESPATH") 767 continue 768 # Defines -- not sure I have comments matching 769 if "defined(" in requirement.lower(): 770 reqdef=requirement.split("(")[1].split(")")[0] 771 if reqdef in self.conf: 772 if isNull: 773 testDict['SKIP'].append("Null requirement not met: "+requirement) 774 continue 775 continue # Success 776 elif not isNull: 777 testDict['SKIP'].append("Required: "+requirement) 778 continue 779 780 # Rest should be packages that we can just get from conf 781 if requirement in ["complex","debug"]: 782 petscconfvar="PETSC_USE_"+requirement.upper() 783 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 784 else: 785 petscconfvar="PETSC_HAVE_"+requirement.upper() 786 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 787 petsccv = self.conf.get(petscconfvar) 788 pkgcv = self.conf.get(pkgconfvar) 789 790 if petsccv or pkgcv: 791 if isNull: 792 if petsccv: 793 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 794 continue 795 else: 796 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 797 continue 798 continue # Success 799 elif not isNull: 800 if not petsccv and not pkgcv: 801 if debug: print("requirement not found: ", requirement) 802 if self.pkg_name == 'petsc': 803 testDict['SKIP'].append(petscconfvar+" requirement not met") 804 else: 805 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 806 continue 807 return testDict['SKIP'] == [] 808 809 def checkOutput(self,exfile,root,srcDict): 810 """ 811 Check and make sure the output files are in the output directory 812 """ 813 debug=False 814 rpath=self.srcrelpath(root) 815 for test in srcDict: 816 if test in self.buildkeys: continue 817 if debug: print(rpath, exfile, test) 818 if 'output_file' in srcDict[test]: 819 output_file=srcDict[test]['output_file'] 820 else: 821 defroot = testparse.getDefaultOutputFileRoot(test) 822 if 'TODO' in srcDict[test]: continue 823 output_file="output/"+defroot+".out" 824 825 fullout=os.path.join(root,output_file) 826 if debug: print("---> ",fullout) 827 if not os.path.exists(fullout): 828 self.missing_files.append(fullout) 829 830 return 831 832 def genPetscTests_summarize(self,dataDict): 833 """ 834 Required method to state what happened 835 """ 836 if not self.summarize: return 837 indent=" " 838 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 839 with open(fhname, "w") as fh: 840 for root in dataDict: 841 relroot=self.srcrelpath(root) 842 pkg=relroot.split("/")[1] 843 fh.write(relroot+"\n") 844 allSrcs=[] 845 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 846 for exfile in dataDict[root]: 847 # Basic information 848 rfile=os.path.join(relroot,exfile) 849 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 850 fh.write(indent+exfile+indent*4+builtStatus+"\n") 851 for test in dataDict[root][exfile]: 852 if test in self.buildkeys: continue 853 line=indent*2+test 854 fh.write(line+"\n") 855 # Looks nice to have the keys in order 856 #for key in dataDict[root][exfile][test]: 857 for key in "isrun abstracted nsize args requires script".split(): 858 if key not in dataDict[root][exfile][test]: continue 859 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 860 fh.write(line+"\n") 861 fh.write("\n") 862 fh.write("\n") 863 fh.write("\n") 864 return 865 866 def genPetscTests(self,root,dirs,files,dataDict): 867 """ 868 Go through and parse the source files in the directory to generate 869 the examples based on the metadata contained in the source files 870 """ 871 debug=False 872 # Use examplesAnalyze to get what the makefles think are sources 873 #self.examplesAnalyze(root,dirs,files,anlzDict) 874 875 data = {} 876 for exfile in files: 877 #TST: Until we replace files, still leaving the originals as is 878 #if not exfile.startswith("new_"+"ex"): continue 879 #if not exfile.startswith("ex"): continue 880 881 # Ignore emacs and other temporary files 882 if exfile.startswith((".", "#")) or exfile.endswith("~"): continue 883 # Only parse source files 884 ext=getlangext(exfile).lstrip('.').replace('.','_') 885 if ext not in LANGS: continue 886 887 # Convenience 888 fullex=os.path.join(root,exfile) 889 if self.verbose: print(' --> '+fullex) 890 data.update(testparse.parseTestFile(fullex,0)) 891 if exfile in data: 892 if self.check_output: 893 self.checkOutput(exfile,root,data[exfile]) 894 else: 895 self.genScriptsAndInfo(exfile,root,data[exfile]) 896 897 dataDict[root] = data 898 return 899 900 def walktree(self,top): 901 """ 902 Walk a directory tree, starting from 'top' 903 """ 904 if self.check_output: 905 print("Checking for missing output files") 906 self.missing_files=[] 907 908 # Goal of action is to fill this dictionary 909 dataDict={} 910 for root, dirs, files in os.walk(top, topdown=True): 911 dirs.sort() 912 files.sort() 913 if "/tests" not in root and "/tutorials" not in root: continue 914 if "dSYM" in root: continue 915 if "tutorials"+os.sep+"build" in root: continue 916 if os.path.basename(root.rstrip("/")) == 'output': continue 917 if self.verbose: print(root) 918 self.genPetscTests(root,dirs,files,dataDict) 919 920 # If checking output, report results 921 if self.check_output: 922 if self.missing_files: 923 for file in set(self.missing_files): # set uniqifies 924 print(file) 925 sys.exit(1) 926 927 # Now summarize this dictionary 928 if self.verbose: self.genPetscTests_summarize(dataDict) 929 return dataDict 930 931 def gen_gnumake(self, fd): 932 """ 933 Overwrite of the method in the base PETSc class 934 """ 935 def write(stem, srcs): 936 for lang in LANGS: 937 if srcs[lang]['srcs']: 938 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 939 for pkg in self.pkg_pkgs: 940 srcs = self.gen_pkg(pkg) 941 write('testsrcs-' + pkg, srcs) 942 # Handle dependencies 943 for lang in LANGS: 944 for exfile in srcs[lang]['srcs']: 945 if exfile in srcs[lang]: 946 ex='$(TESTDIR)/'+getlangsplit(exfile) 947 exfo=ex+'.o' 948 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 949 if deps: 950 # The executable literally depends on the object file because it is linked 951 fd.write(ex +": " + " ".join(deps) +'\n') 952 # The object file containing 'main' does not normally depend on other object 953 # files, but it does when it includes their modules. This dependency is 954 # overly blunt and could be reduced to only depend on object files for 955 # modules that are used, like "*f90aux.o". 956 fd.write(exfo +": " + " ".join(deps) +'\n') 957 958 return self.gendeps 959 960 def gen_pkg(self, pkg): 961 """ 962 Overwrite of the method in the base PETSc class 963 """ 964 return self.sources[pkg] 965 966 def write_gnumake(self, dataDict, output=None): 967 """ 968 Write out something similar to files from gmakegen.py 969 970 Test depends on script which also depends on source 971 file, but since I don't have a good way generating 972 acting on a single file (oops) just depend on 973 executable which in turn will depend on src file 974 """ 975 # Different options for how to set up the targets 976 compileExecsFirst=False 977 978 # Open file 979 with open(output, 'w') as fd: 980 # Write out the sources 981 gendeps = self.gen_gnumake(fd) 982 983 # Write out the tests and execname targets 984 fd.write("\n#Tests and executables\n") # Delimiter 985 986 for pkg in self.pkg_pkgs: 987 # These grab the ones that are built 988 for lang in LANGS: 989 testdeps=[] 990 for ftest in self.tests[pkg][lang]: 991 test=os.path.basename(ftest) 992 basedir=os.path.dirname(ftest) 993 testdeps.append(nameSpace(test,basedir)) 994 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 995 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 996 997 # test targets 998 for ftest in self.tests[pkg][lang]: 999 test=os.path.basename(ftest) 1000 basedir=os.path.dirname(ftest) 1001 testdir="${TESTDIR}/"+basedir+"/" 1002 nmtest=nameSpace(test,basedir) 1003 rundir=os.path.join(testdir,test) 1004 script=test+".sh" 1005 1006 # Deps 1007 exfile=self.tests[pkg][lang][ftest]['exfile'] 1008 fullex=os.path.join(self.srcdir,exfile) 1009 localexec=self.tests[pkg][lang][ftest]['exec'] 1010 execname=os.path.join(testdir,localexec) 1011 fullscript=os.path.join(testdir,script) 1012 tmpfile=os.path.join(testdir,test,test+".tmp") 1013 1014 # *.counts depends on the script and either executable (will 1015 # be run) or the example source file (SKIP or TODO) 1016 fd.write('%s.counts : %s %s' 1017 % (os.path.join('$(TESTDIR)/counts', nmtest), 1018 fullscript, 1019 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1020 ) 1021 if exfile in self.sources[pkg][lang]: 1022 for dep in self.sources[pkg][lang][exfile]: 1023 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1024 fd.write('\n') 1025 1026 # Now write the args: 1027 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1028 1029 return 1030 1031 def write_db(self, dataDict, testdir): 1032 """ 1033 Write out the dataDict into a pickle file 1034 """ 1035 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1036 pickle.dump(dataDict,fd) 1037 return 1038 1039def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1040 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1041 srcdir=None, testdir=None, check=False): 1042 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1043 testdir=os.path.normpath(testdir) 1044 if petsc_arch: 1045 petsc_arch=petsc_arch.rstrip(os.path.sep) 1046 if len(petsc_arch.split(os.path.sep))>1: 1047 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1048 output = os.path.join(testdir, 'testfiles') 1049 1050 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1051 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1052 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1053 testdir=testdir,check=check) 1054 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1055 if not pEx.check_output: 1056 pEx.write_gnumake(dataDict, output) 1057 pEx.write_db(dataDict, testdir) 1058 1059if __name__ == '__main__': 1060 import optparse 1061 parser = optparse.OptionParser() 1062 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1063 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1064 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1065 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1066 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1067 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1068 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1069 help='Check whether output files are in output director') 1070 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1071 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1072 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1073 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1074 1075 opts, extra_args = parser.parse_args() 1076 if extra_args: 1077 import sys 1078 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1079 exit(1) 1080 if opts.testdir is None: 1081 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1082 1083 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1084 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1085 verbose=opts.verbose, 1086 single_ex=opts.single_executable, srcdir=opts.srcdir, 1087 testdir=opts.testdir, check=opts.check_output) 1088