1#!/usr/bin/env python3 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 # Special configuration for CI testing 103 if self.petsc_arch.find('valgrind') >= 0: 104 self.conf['PETSCTEST_VALGRIND']=1 105 106 if self.inInstallDir: 107 # Case 2 discussed above 108 # set PETSC_ARCH to install directory to allow script to work in both 109 dirlist=thisscriptdir.split(os.path.sep) 110 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 111 self.arch_dir=installdir 112 if self.srcdir is None: 113 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 114 else: 115 if petsc_arch == '': 116 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 117 # Case 1 discussed above 118 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 119 if self.srcdir is None: 120 self.srcdir=os.path.join(self.petsc_dir,'src') 121 122 self.testroot_dir=os.path.abspath(testdir) 123 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64 long32 long64".split() 131 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in self.pkg_pkgs: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.relpath(rdir,self.srcdir) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext = getlangext(srcfile) 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 191 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 192 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 193 if srcext in ".cpp".split(): langReq="cpp" 194 if srcext == ".cu": langReq="cu" 195 if srcext == ".c": langReq="c" 196 #if not langReq: print("ERROR: ", srcext, srcfile) 197 return langReq 198 199 def _getAltList(self,output_file,srcdir): 200 ''' Calculate AltList based on output file-- see 201 src/snes/tutorials/output/ex22*.out 202 ''' 203 altlist=[output_file] 204 basefile = getlangsplit(output_file) 205 for i in range(1,9): 206 altroot=basefile+"_alt" 207 if i > 1: altroot=altroot+"_"+str(i) 208 af=altroot+".out" 209 srcaf=os.path.join(srcdir,af) 210 fullaf=os.path.join(self.petsc_dir,srcaf) 211 if os.path.isfile(fullaf): altlist.append(srcaf) 212 213 return altlist 214 215 216 def _getLoopVars(self,inDict,testname, isSubtest=False): 217 """ 218 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 219 Return: 220 inDict['args']: -ksp_monitor 221 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 222 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 223 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 224 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 225 subst should be passed in instead of inDict 226 """ 227 loopVars={}; newargs=[] 228 lsuffix='+' 229 argregex = re.compile(' (?=-[a-zA-Z])') 230 from testparse import parseLoopArgs 231 for key in inDict: 232 if key in ('SKIP', 'regexes'): 233 continue 234 akey=('subargs' if key=='args' else key) # what to assign 235 if akey not in inDict: inDict[akey]='' 236 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 237 # Always generate a loop over nsize, even if there is only one value 238 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 239 keystr = str(inDict[key]) 240 varlist = [] 241 for varset in argregex.split(keystr): 242 if not varset.strip(): continue 243 if '{{' in varset: 244 keyvar,lvars,ftype=parseLoopArgs(varset) 245 if akey not in loopVars: loopVars[akey]={} 246 varlist.append(keyvar) 247 loopVars[akey][keyvar]=[keyvar,lvars] 248 if akey=='nsize': 249 if len(lvars.split()) > 1: 250 lsuffix += akey +'-${i' + keyvar + '}' 251 else: 252 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 253 lsuffix+=keyvar+'-${i' + keyvar + '}_' 254 else: 255 if key=='args': 256 newargs.append(varset.strip()) 257 if varlist: 258 loopVars[akey]['varlist']=varlist 259 260 # For subtests, args are always substituted in (not top level) 261 if isSubtest: 262 inDict['subargs'] += " "+" ".join(newargs) 263 inDict['args']='' 264 if 'label_suffix' in inDict: 265 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 266 else: 267 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 268 else: 269 if loopVars: 270 inDict['args'] = ' '.join(newargs) 271 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 272 return loopVars 273 274 def getArgLabel(self,testDict): 275 """ 276 In all of the arguments in the test dictionary, create a simple 277 string for searching within the makefile system. For simplicity in 278 search, remove "-", for strings, etc. 279 Also, concatenate the arg commands 280 For now, ignore nsize -- seems hard to search for anyway 281 """ 282 # Collect all of the args associated with a test 283 argStr=("" if 'args' not in testDict else testDict['args']) 284 if 'subtests' in testDict: 285 for stest in testDict["subtests"]: 286 sd=testDict[stest] 287 argStr=argStr+("" if 'args' not in sd else sd['args']) 288 289 # Now go through and cleanup 290 argStr=re.sub('{{(.*?)}}',"",argStr) 291 argStr=re.sub('-'," ",argStr) 292 for digit in string.digits: argStr=re.sub(digit," ",argStr) 293 argStr=re.sub("\.","",argStr) 294 argStr=re.sub(",","",argStr) 295 argStr=re.sub('\+',' ',argStr) 296 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 297 return argStr.strip() 298 299 def addToSources(self,exfile,rpath,srcDict): 300 """ 301 Put into data structure that allows easy generation of makefile 302 """ 303 pkg=rpath.split(os.path.sep)[0] 304 relpfile=os.path.join(rpath,exfile) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 if pkg not in self.sources: return 308 self.sources[pkg][lang]['srcs'].append(relpfile) 309 self.sources[pkg][lang][relpfile] = [] 310 if 'depends' in srcDict: 311 depSrcList=srcDict['depends'].split() 312 for depSrc in depSrcList: 313 depObj = getlangsplit(depSrc)+'.o' 314 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 315 316 # In gmakefile, ${TESTDIR} var specifies the object compilation 317 testsdir=rpath+"/" 318 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 319 self.objects[pkg].append(objfile) 320 return 321 322 def addToTests(self,test,rpath,exfile,execname,testDict): 323 """ 324 Put into data structure that allows easy generation of makefile 325 Organized by languages to allow testing of languages 326 """ 327 pkg=rpath.split("/")[0] 328 nmtest=os.path.join(rpath,test) 329 lang=self.getLanguage(exfile) 330 if not lang: return 331 if pkg not in self.tests: return 332 self.tests[pkg][lang][nmtest]={} 333 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 334 self.tests[pkg][lang][nmtest]['exec']=execname 335 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 336 return 337 338 def getExecname(self,exfile,rpath): 339 """ 340 Generate bash script using template found next to this file. 341 This file is read in at constructor time to avoid file I/O 342 """ 343 if self.single_ex: 344 execname=rpath.split("/")[1]+"-ex" 345 else: 346 execname=getlangsplit(exfile) 347 return execname 348 349 def getSubstVars(self,testDict,rpath,testname): 350 """ 351 Create a dictionary with all of the variables that get substituted 352 into the template commands found in example_template.py 353 """ 354 subst={} 355 356 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 357 if 'nsize' not in testDict: testDict['nsize'] = '1' 358 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 359 for ak in testparse.acceptedkeys: 360 if ak=='test': continue 361 subst[ak]=(testDict[ak] if ak in testDict else '') 362 363 # Now do other variables 364 subst['execname']=testDict['execname'] 365 subst['error']='' 366 if 'filter' in testDict: 367 if testDict['filter'].startswith("Error:"): 368 subst['error']="Error" 369 subst['filter']=testDict['filter'].lstrip("Error:") 370 else: 371 subst['filter']=testDict['filter'] 372 373 # Others 374 subst['subargs']='' # Default. For variables override 375 subst['srcdir']=os.path.join(self.srcdir, rpath) 376 subst['label_suffix']='' 377 subst['comments']="\n#".join(subst['comments'].split("\n")) 378 if subst['comments']: subst['comments']="#"+subst['comments'] 379 subst['exec']="../"+subst['execname'] 380 subst['testroot']=self.testroot_dir 381 subst['testname']=testname 382 dp = self.conf.get('DATAFILESPATH','') 383 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 384 385 # This is used to label some matrices 386 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 387 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 388 389 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 390 391 #Conf vars 392 if self.petsc_arch.find('valgrind')>=0: 393 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 394 else: 395 subst['mpiexec']=self.conf['MPIEXEC'] 396 subst['pkg_name']=self.pkg_name 397 subst['pkg_dir']=self.pkg_dir 398 subst['pkg_arch']=self.petsc_arch 399 subst['CONFIG_DIR']=thisscriptdir 400 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 401 subst['diff']=self.conf['DIFF'] 402 subst['rm']=self.conf['RM'] 403 subst['grep']=self.conf['GREP'] 404 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 405 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 406 407 # Output file is special because of subtests override 408 defroot = testparse.getDefaultOutputFileRoot(testname) 409 if 'output_file' not in testDict: 410 subst['output_file']="output/"+defroot+".out" 411 subst['redirect_file']=defroot+".tmp" 412 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 413 414 # Add in the full path here. 415 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 416 417 subst['regexes']={} 418 for subkey in subst: 419 if subkey=='regexes': continue 420 if not isinstance(subst[subkey],str): continue 421 patt="@"+subkey.upper()+"@" 422 subst['regexes'][subkey]=re.compile(patt) 423 424 return subst 425 426 def _substVars(self,subst,origStr): 427 """ 428 Substitute variables 429 """ 430 Str=origStr 431 for subkey in subst: 432 if subkey=='regexes': continue 433 if not isinstance(subst[subkey],str): continue 434 if subkey.upper() not in Str: continue 435 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 436 return Str 437 438 def getCmds(self,subst,i, debug=False): 439 """ 440 Generate bash script using template found next to this file. 441 This file is read in at constructor time to avoid file I/O 442 """ 443 nindnt=i # the start and has to be consistent with below 444 cmdindnt=self.indent*nindnt 445 cmdLines="" 446 447 # MPI is the default -- but we have a few odd commands 448 if not subst['command']: 449 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 450 else: 451 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 452 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 453 454 cmdLines+=cmdindnt+'if test $res = 0; then\n' 455 diffindnt=self.indent*(nindnt+1) 456 457 # Do some checks on existence of output_file and alt files 458 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 459 if not subst['TODO']: 460 print("Warning: "+subst['output_file']+" not found.") 461 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 462 463 # altlist always has output_file 464 if len(altlist)==1: 465 cmd=diffindnt+self._substVars(subst,example_template.difftest) 466 else: 467 if debug: print("Found alt files: ",altlist) 468 # Have to do it by hand a bit because of variable number of alt files 469 rf=subst['redirect_file'] 470 cmd=diffindnt+example_template.difftest.split('@')[0] 471 for i in range(len(altlist)): 472 af=altlist[i] 473 cmd+=af+' '+rf 474 if i!=len(altlist)-1: 475 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 476 cmd+=' || ${diff_exe} ' 477 else: 478 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 479 cmd+=subst['label_suffix']+' ""' # Quotes are painful 480 cmdLines+=cmd+"\n" 481 cmdLines+=cmdindnt+'else\n' 482 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 483 cmdLines+=cmdindnt+'fi\n' 484 return cmdLines 485 486 def _writeTodoSkip(self,fh,tors,reasons,footer): 487 """ 488 Write out the TODO and SKIP lines in the file 489 The TODO or SKIP variable, tors, should be lower case 490 """ 491 TORS=tors.upper() 492 template=eval("example_template."+tors+"line") 493 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 494 tab = '' 495 if reasons: 496 fh.write('if ! $force; then\n') 497 tab = tab + ' ' 498 if reasons == ["Requires DATAFILESPATH"]: 499 # The only reason not to run is DATAFILESPATH, which we check at run-time 500 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 501 tab = tab + ' ' 502 if reasons: 503 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 504 fh.write(tab+footer+"\n") 505 fh.write(tab+"exit\n") 506 if reasons == ["Requires DATAFILESPATH"]: 507 fh.write(' fi\n') 508 if reasons: 509 fh.write('fi\n') 510 fh.write('\n\n') 511 return 512 513 def getLoopVarsHead(self,loopVars,i,usedVars={}): 514 """ 515 Generate a nicely indented string with the format loops 516 Here is what the data structure looks like 517 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 518 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 519 loopVars['subargs']['pc_type']=["j","cholesky sor"] 520 """ 521 outstr=''; indnt=self.indent 522 523 for key in loopVars: 524 for var in loopVars[key]['varlist']: 525 varval=loopVars[key][var] 526 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 527 outstr += "\n\n" 528 529 for key in loopVars: 530 for var in loopVars[key]['varlist']: 531 varval=loopVars[key][var] 532 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 533 i = i + 1 534 return (outstr,i) 535 536 def getLoopVarsFoot(self,loopVars,i): 537 outstr=''; indnt=self.indent 538 for key in loopVars: 539 for var in loopVars[key]['varlist']: 540 i = i - 1 541 outstr += indnt * i + "done\n" 542 return (outstr,i) 543 544 def genRunScript(self,testname,root,isRun,srcDict): 545 """ 546 Generate bash script using template found next to this file. 547 This file is read in at constructor time to avoid file I/O 548 """ 549 # runscript_dir directory has to be consistent with gmakefile 550 testDict=srcDict[testname] 551 rpath=self.srcrelpath(root) 552 runscript_dir=os.path.join(self.testroot_dir,rpath) 553 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 554 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 555 556 # Get variables to go into shell scripts. last time testDict used 557 subst=self.getSubstVars(testDict,rpath,testname) 558 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 559 if 'subtests' in testDict: 560 # The subtests inherit inDict, so we don't need top-level loops. 561 loopVars = {} 562 563 #Handle runfiles 564 for lfile in subst.get('localrunfiles','').split(): 565 install_files(os.path.join(root, lfile), 566 os.path.join(runscript_dir, os.path.dirname(lfile))) 567 # Check subtests for local runfiles 568 for stest in subst.get("subtests",[]): 569 for lfile in testDict[stest].get('localrunfiles','').split(): 570 install_files(os.path.join(root, lfile), 571 os.path.join(runscript_dir, os.path.dirname(lfile))) 572 573 # Now substitute the key variables into the header and footer 574 header=self._substVars(subst,example_template.header) 575 # The header is done twice to enable @...@ in header 576 header=self._substVars(subst,header) 577 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 578 579 # Start writing the file 580 fh.write(header+"\n") 581 582 # If there is a TODO or a SKIP then we do it before writing out the 583 # rest of the command (which is useful for working on the test) 584 # SKIP and TODO can be for the source file or for the runs 585 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 586 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 587 588 j=0 # for indentation 589 590 if loopVars: 591 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 592 if (loopHead): fh.write(loopHead+"\n") 593 594 # Subtests are special 595 allLoopVars=list(loopVars.keys()) 596 if 'subtests' in testDict: 597 substP=subst # Subtests can inherit args but be careful 598 k=0 # for label suffixes 599 for stest in testDict["subtests"]: 600 subst=substP.copy() 601 subst.update(testDict[stest]) 602 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 603 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 604 if sLoopVars: 605 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 606 allLoopVars+=list(sLoopVars.keys()) 607 fh.write(sLoopHead+"\n") 608 fh.write(self.getCmds(subst,j)+"\n") 609 if sLoopVars: 610 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 611 fh.write(sLoopFoot+"\n") 612 else: 613 fh.write(self.getCmds(subst,j)+"\n") 614 615 if loopVars: 616 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 617 fh.write(loopFoot+"\n") 618 619 fh.write(footer+"\n") 620 621 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 622 #if '10_9' in testname: sys.exit() 623 return 624 625 def genScriptsAndInfo(self,exfile,root,srcDict): 626 """ 627 Generate scripts from the source file, determine if built, etc. 628 For every test in the exfile with info in the srcDict: 629 1. Determine if it needs to be run for this arch 630 2. Generate the script 631 3. Generate the data needed to write out the makefile in a 632 convenient way 633 All tests are *always* run, but some may be SKIP'd per the TAP standard 634 """ 635 debug=False 636 rpath=self.srcrelpath(root) 637 execname=self.getExecname(exfile,rpath) 638 isBuilt=self._isBuilt(exfile,srcDict) 639 for test in srcDict: 640 if test in self.buildkeys: continue 641 if debug: print(nameSpace(exfile,root), test) 642 srcDict[test]['execname']=execname # Convenience in generating scripts 643 isRun=self._isRun(srcDict[test]) 644 self.genRunScript(test,root,isRun,srcDict) 645 srcDict[test]['isrun']=isRun 646 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 647 648 # This adds to datastructure for building deps 649 if isBuilt: self.addToSources(exfile,rpath,srcDict) 650 return 651 652 def _isBuilt(self,exfile,srcDict): 653 """ 654 Determine if this file should be built. 655 """ 656 # Get the language based on file extension 657 srcDict['SKIP'] = [] 658 lang=self.getLanguage(exfile) 659 if (lang=="F" or lang=="F90"): 660 if not self.have_fortran: 661 srcDict["SKIP"].append("Fortran required for this test") 662 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 663 srcDict["SKIP"].append("Fortran f90freeform required for this test") 664 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 665 srcDict["SKIP"].append("CUDA required for this test") 666 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 667 srcDict["SKIP"].append("HIP required for this test") 668 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 669 srcDict["SKIP"].append("SYCL required for this test") 670 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 671 srcDict["SKIP"].append("KOKKOS required for this test") 672 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 673 srcDict["SKIP"].append("RAJA required for this test") 674 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 675 srcDict["SKIP"].append("C++ required for this test") 676 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 677 srcDict["SKIP"].append("C++ required for this test") 678 679 # Deprecated source files 680 if srcDict.get("TODO"): 681 return False 682 683 # isRun can work with srcDict to handle the requires 684 if "requires" in srcDict: 685 if srcDict["requires"]: 686 return self._isRun(srcDict) 687 688 return srcDict['SKIP'] == [] 689 690 691 def _isRun(self,testDict, debug=False): 692 """ 693 Based on the requirements listed in the src file and the petscconf.h 694 info, determine whether this test should be run or not. 695 """ 696 indent=" " 697 698 if 'SKIP' not in testDict: 699 testDict['SKIP'] = [] 700 # MPI requirements 701 if 'MPI_IS_MPIUNI' in self.conf: 702 if testDict.get('nsize', '1') != '1': 703 testDict['SKIP'].append("Parallel test with serial build") 704 705 # The requirements for the test are the sum of all the run subtests 706 if 'subtests' in testDict: 707 if 'requires' not in testDict: testDict['requires']="" 708 for stest in testDict['subtests']: 709 if 'requires' in testDict[stest]: 710 testDict['requires']+=" "+testDict[stest]['requires'] 711 if testDict[stest].get('nsize', '1') != '1': 712 testDict['SKIP'].append("Parallel test with serial build") 713 break 714 715 # Now go through all requirements 716 if 'requires' in testDict: 717 for requirement in testDict['requires'].split(): 718 requirement=requirement.strip() 719 if not requirement: continue 720 if debug: print(indent+"Requirement: ", requirement) 721 isNull=False 722 if requirement.startswith("!"): 723 requirement=requirement[1:]; isNull=True 724 # Precision requirement for reals 725 if requirement in self.precision_types: 726 if self.conf['PETSC_PRECISION']==requirement: 727 if isNull: 728 testDict['SKIP'].append("not "+requirement+" required") 729 continue 730 continue # Success 731 elif not isNull: 732 testDict['SKIP'].append(requirement+" required") 733 continue 734 # Precision requirement for ints 735 if requirement in self.integer_types: 736 if requirement=="int32": 737 if self.conf['PETSC_SIZEOF_INT']==4: 738 if isNull: 739 testDict['SKIP'].append("not int32 required") 740 continue 741 continue # Success 742 elif not isNull: 743 testDict['SKIP'].append("int32 required") 744 continue 745 if requirement=="int64": 746 if self.conf['PETSC_SIZEOF_INT']==8: 747 if isNull: 748 testDict['SKIP'].append("NOT int64 required") 749 continue 750 continue # Success 751 elif not isNull: 752 testDict['SKIP'].append("int64 required") 753 continue 754 if requirement.startswith("long"): 755 reqsize = int(requirement[4:])//8 756 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 757 if longsize==reqsize: 758 if isNull: 759 testDict['SKIP'].append("not %s required" % requirement) 760 continue 761 continue # Success 762 elif not isNull: 763 testDict['SKIP'].append("%s required" % requirement) 764 continue 765 # Datafilespath 766 if requirement=="datafilespath" and not isNull: 767 testDict['SKIP'].append("Requires DATAFILESPATH") 768 continue 769 # Defines -- not sure I have comments matching 770 if "defined(" in requirement.lower(): 771 reqdef=requirement.split("(")[1].split(")")[0] 772 if reqdef in self.conf: 773 if isNull: 774 testDict['SKIP'].append("Null requirement not met: "+requirement) 775 continue 776 continue # Success 777 elif not isNull: 778 testDict['SKIP'].append("Required: "+requirement) 779 continue 780 781 # Rest should be packages that we can just get from conf 782 if requirement in ["complex","debug"]: 783 petscconfvar="PETSC_USE_"+requirement.upper() 784 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 785 else: 786 petscconfvar="PETSC_HAVE_"+requirement.upper() 787 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 788 petsccv = self.conf.get(petscconfvar) 789 pkgcv = self.conf.get(pkgconfvar) 790 791 if petsccv or pkgcv: 792 if isNull: 793 if petsccv: 794 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 795 continue 796 else: 797 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 798 continue 799 continue # Success 800 elif not isNull: 801 if not petsccv and not pkgcv: 802 if debug: print("requirement not found: ", requirement) 803 if self.pkg_name == 'petsc': 804 testDict['SKIP'].append(petscconfvar+" requirement not met") 805 else: 806 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 807 continue 808 return testDict['SKIP'] == [] 809 810 def checkOutput(self,exfile,root,srcDict): 811 """ 812 Check and make sure the output files are in the output directory 813 """ 814 debug=False 815 rpath=self.srcrelpath(root) 816 for test in srcDict: 817 if test in self.buildkeys: continue 818 if debug: print(rpath, exfile, test) 819 if 'output_file' in srcDict[test]: 820 output_file=srcDict[test]['output_file'] 821 else: 822 defroot = testparse.getDefaultOutputFileRoot(test) 823 if 'TODO' in srcDict[test]: continue 824 output_file="output/"+defroot+".out" 825 826 fullout=os.path.join(root,output_file) 827 if debug: print("---> ",fullout) 828 if not os.path.exists(fullout): 829 self.missing_files.append(fullout) 830 831 return 832 833 def genPetscTests_summarize(self,dataDict): 834 """ 835 Required method to state what happened 836 """ 837 if not self.summarize: return 838 indent=" " 839 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 840 with open(fhname, "w") as fh: 841 for root in dataDict: 842 relroot=self.srcrelpath(root) 843 pkg=relroot.split("/")[1] 844 fh.write(relroot+"\n") 845 allSrcs=[] 846 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 847 for exfile in dataDict[root]: 848 # Basic information 849 rfile=os.path.join(relroot,exfile) 850 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 851 fh.write(indent+exfile+indent*4+builtStatus+"\n") 852 for test in dataDict[root][exfile]: 853 if test in self.buildkeys: continue 854 line=indent*2+test 855 fh.write(line+"\n") 856 # Looks nice to have the keys in order 857 #for key in dataDict[root][exfile][test]: 858 for key in "isrun abstracted nsize args requires script".split(): 859 if key not in dataDict[root][exfile][test]: continue 860 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 861 fh.write(line+"\n") 862 fh.write("\n") 863 fh.write("\n") 864 fh.write("\n") 865 return 866 867 def genPetscTests(self,root,dirs,files,dataDict): 868 """ 869 Go through and parse the source files in the directory to generate 870 the examples based on the metadata contained in the source files 871 """ 872 debug=False 873 # Use examplesAnalyze to get what the makefles think are sources 874 #self.examplesAnalyze(root,dirs,files,anlzDict) 875 876 dataDict[root]={} 877 878 for exfile in files: 879 #TST: Until we replace files, still leaving the orginals as is 880 #if not exfile.startswith("new_"+"ex"): continue 881 #if not exfile.startswith("ex"): continue 882 883 # Ignore emacs and other temporary files 884 if exfile.startswith("."): continue 885 if exfile.startswith("#"): continue 886 if exfile.endswith("~"): continue 887 # Only parse source files 888 ext=getlangext(exfile).lstrip('.').replace('.','_') 889 if ext not in LANGS: continue 890 891 # Convenience 892 fullex=os.path.join(root,exfile) 893 if self.verbose: print(' --> '+fullex) 894 dataDict[root].update(testparse.parseTestFile(fullex,0)) 895 if exfile in dataDict[root]: 896 if not self.check_output: 897 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 898 else: 899 self.checkOutput(exfile,root,dataDict[root][exfile]) 900 901 return 902 903 def walktree(self,top): 904 """ 905 Walk a directory tree, starting from 'top' 906 """ 907 if self.check_output: 908 print("Checking for missing output files") 909 self.missing_files=[] 910 911 # Goal of action is to fill this dictionary 912 dataDict={} 913 for root, dirs, files in os.walk(top, topdown=True): 914 dirs.sort() 915 files.sort() 916 if "/tests" not in root and "/tutorials" not in root: continue 917 if "dSYM" in root: continue 918 if "tutorials"+os.sep+"build" in root: continue 919 if os.path.basename(root.rstrip("/")) == 'output': continue 920 if self.verbose: print(root) 921 self.genPetscTests(root,dirs,files,dataDict) 922 923 # If checking output, report results 924 if self.check_output: 925 if self.missing_files: 926 for file in set(self.missing_files): # set uniqifies 927 print(file) 928 sys.exit(1) 929 930 # Now summarize this dictionary 931 if self.verbose: self.genPetscTests_summarize(dataDict) 932 return dataDict 933 934 def gen_gnumake(self, fd): 935 """ 936 Overwrite of the method in the base PETSc class 937 """ 938 def write(stem, srcs): 939 for lang in LANGS: 940 if srcs[lang]['srcs']: 941 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 942 for pkg in self.pkg_pkgs: 943 srcs = self.gen_pkg(pkg) 944 write('testsrcs-' + pkg, srcs) 945 # Handle dependencies 946 for lang in LANGS: 947 for exfile in srcs[lang]['srcs']: 948 if exfile in srcs[lang]: 949 ex='$(TESTDIR)/'+getlangsplit(exfile) 950 exfo=ex+'.o' 951 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 952 if deps: 953 # The executable literally depends on the object file because it is linked 954 fd.write(ex +": " + " ".join(deps) +'\n') 955 # The object file containing 'main' does not normally depend on other object 956 # files, but it does when it includes their modules. This dependency is 957 # overly blunt and could be reduced to only depend on object files for 958 # modules that are used, like "*f90aux.o". 959 fd.write(exfo +": " + " ".join(deps) +'\n') 960 961 return self.gendeps 962 963 def gen_pkg(self, pkg): 964 """ 965 Overwrite of the method in the base PETSc class 966 """ 967 return self.sources[pkg] 968 969 def write_gnumake(self, dataDict, output=None): 970 """ 971 Write out something similar to files from gmakegen.py 972 973 Test depends on script which also depends on source 974 file, but since I don't have a good way generating 975 acting on a single file (oops) just depend on 976 executable which in turn will depend on src file 977 """ 978 # Different options for how to set up the targets 979 compileExecsFirst=False 980 981 # Open file 982 with open(output, 'w') as fd: 983 # Write out the sources 984 gendeps = self.gen_gnumake(fd) 985 986 # Write out the tests and execname targets 987 fd.write("\n#Tests and executables\n") # Delimiter 988 989 for pkg in self.pkg_pkgs: 990 # These grab the ones that are built 991 for lang in LANGS: 992 testdeps=[] 993 for ftest in self.tests[pkg][lang]: 994 test=os.path.basename(ftest) 995 basedir=os.path.dirname(ftest) 996 testdeps.append(nameSpace(test,basedir)) 997 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 998 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 999 1000 # test targets 1001 for ftest in self.tests[pkg][lang]: 1002 test=os.path.basename(ftest) 1003 basedir=os.path.dirname(ftest) 1004 testdir="${TESTDIR}/"+basedir+"/" 1005 nmtest=nameSpace(test,basedir) 1006 rundir=os.path.join(testdir,test) 1007 script=test+".sh" 1008 1009 # Deps 1010 exfile=self.tests[pkg][lang][ftest]['exfile'] 1011 fullex=os.path.join(self.srcdir,exfile) 1012 localexec=self.tests[pkg][lang][ftest]['exec'] 1013 execname=os.path.join(testdir,localexec) 1014 fullscript=os.path.join(testdir,script) 1015 tmpfile=os.path.join(testdir,test,test+".tmp") 1016 1017 # *.counts depends on the script and either executable (will 1018 # be run) or the example source file (SKIP or TODO) 1019 fd.write('%s.counts : %s %s' 1020 % (os.path.join('$(TESTDIR)/counts', nmtest), 1021 fullscript, 1022 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1023 ) 1024 if exfile in self.sources[pkg][lang]: 1025 for dep in self.sources[pkg][lang][exfile]: 1026 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1027 fd.write('\n') 1028 1029 # Now write the args: 1030 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1031 1032 return 1033 1034 def write_db(self, dataDict, testdir): 1035 """ 1036 Write out the dataDict into a pickle file 1037 """ 1038 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1039 pickle.dump(dataDict,fd) 1040 return 1041 1042def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1043 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1044 srcdir=None, testdir=None, check=False): 1045 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1046 testdir=os.path.normpath(testdir) 1047 if petsc_arch: 1048 petsc_arch=petsc_arch.rstrip(os.path.sep) 1049 if len(petsc_arch.split(os.path.sep))>1: 1050 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1051 output = os.path.join(testdir, 'testfiles') 1052 1053 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1054 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1055 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1056 testdir=testdir,check=check) 1057 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1058 if not pEx.check_output: 1059 pEx.write_gnumake(dataDict, output) 1060 pEx.write_db(dataDict, testdir) 1061 1062if __name__ == '__main__': 1063 import optparse 1064 parser = optparse.OptionParser() 1065 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1066 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1067 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1068 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1069 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1070 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1071 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1072 help='Check whether output files are in output director') 1073 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1074 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1075 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1076 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1077 1078 opts, extra_args = parser.parse_args() 1079 if extra_args: 1080 import sys 1081 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1082 exit(1) 1083 if opts.testdir is None: 1084 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1085 1086 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1087 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1088 verbose=opts.verbose, 1089 single_ex=opts.single_executable, srcdir=opts.srcdir, 1090 testdir=opts.testdir, check=opts.check_output) 1091