1#!/usr/bin/env python 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 # Special configuration for CI testing 103 if self.petsc_arch.find('valgrind') >= 0: 104 self.conf['PETSCTEST_VALGRIND']=1 105 106 if self.inInstallDir: 107 # Case 2 discussed above 108 # set PETSC_ARCH to install directory to allow script to work in both 109 dirlist=thisscriptdir.split(os.path.sep) 110 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 111 self.arch_dir=installdir 112 if self.srcdir is None: 113 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 114 else: 115 if petsc_arch == '': 116 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 117 # Case 1 discussed above 118 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 119 if self.srcdir is None: 120 self.srcdir=os.path.join(self.petsc_dir,'src') 121 122 self.testroot_dir=os.path.abspath(testdir) 123 124 self.verbose=verbose 125 # Whether to write out a useful debugging 126 self.summarize=True if verbose else False 127 128 # For help in setting the requirements 129 self.precision_types="single double __float128 int32".split() 130 self.integer_types="int32 int64 long32 long64".split() 131 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 132 133 # Things that are not test 134 self.buildkeys=testparse.buildkeys 135 136 # Adding a dictionary for storing sources, objects, and tests 137 # to make building the dependency tree easier 138 self.sources={} 139 self.objects={} 140 self.tests={} 141 for pkg in self.pkg_pkgs: 142 self.sources[pkg]={} 143 self.objects[pkg]=[] 144 self.tests[pkg]={} 145 for lang in LANGS: 146 self.sources[pkg][lang]={} 147 self.sources[pkg][lang]['srcs']=[] 148 self.tests[pkg][lang]={} 149 150 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 151 152 self.indent=" " 153 if self.verbose: print('Finishing the constructor') 154 return 155 156 def srcrelpath(self,rdir): 157 """ 158 Get relative path to source directory 159 """ 160 return os.path.relpath(rdir,self.srcdir) 161 162 def getInInstallDir(self,thisscriptdir): 163 """ 164 When petsc is installed then this file in installed in: 165 <PREFIX>/share/petsc/examples/config/gmakegentest.py 166 otherwise the path is: 167 <PETSC_DIR>/config/gmakegentest.py 168 We use this difference to determine if we are in installdir 169 """ 170 dirlist=thisscriptdir.split(os.path.sep) 171 if len(dirlist)>4: 172 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 173 if lastfour==os.path.join('share','petsc','examples','config'): 174 return True 175 else: 176 return False 177 else: 178 return False 179 180 def getLanguage(self,srcfile): 181 """ 182 Based on the source, determine associated language as found in gmakegen.LANGS 183 Can we just return srcext[1:\] now? 184 """ 185 langReq=None 186 srcext = getlangext(srcfile) 187 if srcext in ".F90".split(): langReq="F90" 188 if srcext in ".F".split(): langReq="F" 189 if srcext in ".cxx".split(): langReq="cxx" 190 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 191 if srcext in ".hip.cpp".split(): langReq="hip_cpp" 192 if srcext in ".raja.cxx".split(): langReq="raja_cxx" 193 if srcext in ".cpp".split(): langReq="cpp" 194 if srcext == ".cu": langReq="cu" 195 if srcext == ".c": langReq="c" 196 #if not langReq: print("ERROR: ", srcext, srcfile) 197 return langReq 198 199 def _getAltList(self,output_file,srcdir): 200 ''' Calculate AltList based on output file-- see 201 src/snes/tutorials/output/ex22*.out 202 ''' 203 altlist=[output_file] 204 basefile = getlangsplit(output_file) 205 for i in range(1,9): 206 altroot=basefile+"_alt" 207 if i > 1: altroot=altroot+"_"+str(i) 208 af=altroot+".out" 209 srcaf=os.path.join(srcdir,af) 210 fullaf=os.path.join(self.petsc_dir,srcaf) 211 if os.path.isfile(fullaf): altlist.append(srcaf) 212 213 return altlist 214 215 216 def _getLoopVars(self,inDict,testname, isSubtest=False): 217 """ 218 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 219 Return: 220 inDict['args']: -ksp_monitor 221 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 222 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 223 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 224 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 225 subst should be passed in instead of inDict 226 """ 227 loopVars={}; newargs=[] 228 lsuffix='+' 229 argregex = re.compile(' (?=-[a-zA-Z])') 230 from testparse import parseLoopArgs 231 for key in inDict: 232 if key in ('SKIP', 'regexes'): 233 continue 234 akey=('subargs' if key=='args' else key) # what to assign 235 if akey not in inDict: inDict[akey]='' 236 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 237 # Always generate a loop over nsize, even if there is only one value 238 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 239 keystr = str(inDict[key]) 240 varlist = [] 241 for varset in argregex.split(keystr): 242 if not varset.strip(): continue 243 if '{{' in varset: 244 keyvar,lvars,ftype=parseLoopArgs(varset) 245 if akey not in loopVars: loopVars[akey]={} 246 varlist.append(keyvar) 247 loopVars[akey][keyvar]=[keyvar,lvars] 248 if akey=='nsize': 249 if len(lvars.split()) > 1: 250 lsuffix += akey +'-${i' + keyvar + '}' 251 else: 252 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 253 lsuffix+=keyvar+'-${i' + keyvar + '}_' 254 else: 255 if key=='args': 256 newargs.append(varset.strip()) 257 if varlist: 258 loopVars[akey]['varlist']=varlist 259 260 # For subtests, args are always substituted in (not top level) 261 if isSubtest: 262 inDict['subargs'] += " "+" ".join(newargs) 263 inDict['args']='' 264 if 'label_suffix' in inDict: 265 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 266 else: 267 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 268 else: 269 if loopVars: 270 inDict['args'] = ' '.join(newargs) 271 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 272 return loopVars 273 274 def getArgLabel(self,testDict): 275 """ 276 In all of the arguments in the test dictionary, create a simple 277 string for searching within the makefile system. For simplicity in 278 search, remove "-", for strings, etc. 279 Also, concatenate the arg commands 280 For now, ignore nsize -- seems hard to search for anyway 281 """ 282 # Collect all of the args associated with a test 283 argStr=("" if 'args' not in testDict else testDict['args']) 284 if 'subtests' in testDict: 285 for stest in testDict["subtests"]: 286 sd=testDict[stest] 287 argStr=argStr+("" if 'args' not in sd else sd['args']) 288 289 # Now go through and cleanup 290 argStr=re.sub('{{(.*?)}}',"",argStr) 291 argStr=re.sub('-'," ",argStr) 292 for digit in string.digits: argStr=re.sub(digit," ",argStr) 293 argStr=re.sub("\.","",argStr) 294 argStr=re.sub(",","",argStr) 295 argStr=re.sub('\+',' ',argStr) 296 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 297 return argStr.strip() 298 299 def addToSources(self,exfile,rpath,srcDict): 300 """ 301 Put into data structure that allows easy generation of makefile 302 """ 303 pkg=rpath.split(os.path.sep)[0] 304 relpfile=os.path.join(rpath,exfile) 305 lang=self.getLanguage(exfile) 306 if not lang: return 307 if pkg not in self.sources: return 308 self.sources[pkg][lang]['srcs'].append(relpfile) 309 self.sources[pkg][lang][relpfile] = [] 310 if 'depends' in srcDict: 311 depSrcList=srcDict['depends'].split() 312 for depSrc in depSrcList: 313 depObj = getlangsplit(depSrc)+'.o' 314 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 315 316 # In gmakefile, ${TESTDIR} var specifies the object compilation 317 testsdir=rpath+"/" 318 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 319 self.objects[pkg].append(objfile) 320 return 321 322 def addToTests(self,test,rpath,exfile,execname,testDict): 323 """ 324 Put into data structure that allows easy generation of makefile 325 Organized by languages to allow testing of languages 326 """ 327 pkg=rpath.split("/")[0] 328 nmtest=os.path.join(rpath,test) 329 lang=self.getLanguage(exfile) 330 if not lang: return 331 if pkg not in self.tests: return 332 self.tests[pkg][lang][nmtest]={} 333 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 334 self.tests[pkg][lang][nmtest]['exec']=execname 335 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 336 return 337 338 def getExecname(self,exfile,rpath): 339 """ 340 Generate bash script using template found next to this file. 341 This file is read in at constructor time to avoid file I/O 342 """ 343 if self.single_ex: 344 execname=rpath.split("/")[1]+"-ex" 345 else: 346 execname=getlangsplit(exfile) 347 return execname 348 349 def getSubstVars(self,testDict,rpath,testname): 350 """ 351 Create a dictionary with all of the variables that get substituted 352 into the template commands found in example_template.py 353 """ 354 subst={} 355 356 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 357 if 'nsize' not in testDict: testDict['nsize'] = '1' 358 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 359 for ak in testparse.acceptedkeys: 360 if ak=='test': continue 361 subst[ak]=(testDict[ak] if ak in testDict else '') 362 363 # Now do other variables 364 subst['execname']=testDict['execname'] 365 subst['error']='' 366 if 'filter' in testDict: 367 if testDict['filter'].startswith("Error:"): 368 subst['error']="Error" 369 subst['filter']=testDict['filter'].lstrip("Error:") 370 else: 371 subst['filter']=testDict['filter'] 372 373 # Others 374 subst['subargs']='' # Default. For variables override 375 subst['srcdir']=os.path.join(self.srcdir, rpath) 376 subst['label_suffix']='' 377 subst['comments']="\n#".join(subst['comments'].split("\n")) 378 if subst['comments']: subst['comments']="#"+subst['comments'] 379 subst['exec']="../"+subst['execname'] 380 subst['testroot']=self.testroot_dir 381 subst['testname']=testname 382 dp = self.conf.get('DATAFILESPATH','') 383 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 384 385 # This is used to label some matrices 386 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 387 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 388 389 if 'PETSC_OPTIONS' in os.environ: 390 subst['petsc_options']=os.environ['PETSC_OPTIONS'] 391 else: 392 subst['petsc_options']='' 393 subst['petsc_test_options']=self.conf['PETSC_TEST_OPTIONS'] 394 395 #Conf vars 396 if self.petsc_arch.find('valgrind')>=0: 397 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 398 else: 399 subst['mpiexec']=self.conf['MPIEXEC'] 400 subst['pkg_name']=self.pkg_name 401 subst['pkg_dir']=self.pkg_dir 402 subst['pkg_arch']=self.petsc_arch 403 subst['CONFIG_DIR']=thisscriptdir 404 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 405 subst['diff']=self.conf['DIFF'] 406 subst['rm']=self.conf['RM'] 407 subst['grep']=self.conf['GREP'] 408 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 409 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 410 411 # Output file is special because of subtests override 412 defroot = testparse.getDefaultOutputFileRoot(testname) 413 if 'output_file' not in testDict: 414 subst['output_file']="output/"+defroot+".out" 415 subst['redirect_file']=defroot+".tmp" 416 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 417 418 # Add in the full path here. 419 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 420 421 subst['regexes']={} 422 for subkey in subst: 423 if subkey=='regexes': continue 424 if not isinstance(subst[subkey],str): continue 425 patt="@"+subkey.upper()+"@" 426 subst['regexes'][subkey]=re.compile(patt) 427 428 return subst 429 430 def _substVars(self,subst,origStr): 431 """ 432 Substitute variables 433 """ 434 Str=origStr 435 for subkey in subst: 436 if subkey=='regexes': continue 437 if not isinstance(subst[subkey],str): continue 438 if subkey.upper() not in Str: continue 439 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 440 return Str 441 442 def getCmds(self,subst,i, debug=False): 443 """ 444 Generate bash script using template found next to this file. 445 This file is read in at constructor time to avoid file I/O 446 """ 447 nindnt=i # the start and has to be consistent with below 448 cmdindnt=self.indent*nindnt 449 cmdLines="" 450 451 # MPI is the default -- but we have a few odd commands 452 if not subst['command']: 453 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 454 else: 455 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 456 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 457 458 cmdLines+=cmdindnt+'if test $res = 0; then\n' 459 diffindnt=self.indent*(nindnt+1) 460 461 # Do some checks on existence of output_file and alt files 462 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 463 if not subst['TODO']: 464 print("Warning: "+subst['output_file']+" not found.") 465 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 466 467 # altlist always has output_file 468 if len(altlist)==1: 469 cmd=diffindnt+self._substVars(subst,example_template.difftest) 470 else: 471 if debug: print("Found alt files: ",altlist) 472 # Have to do it by hand a bit because of variable number of alt files 473 rf=subst['redirect_file'] 474 cmd=diffindnt+example_template.difftest.split('@')[0] 475 for i in range(len(altlist)): 476 af=altlist[i] 477 cmd+=af+' '+rf 478 if i!=len(altlist)-1: 479 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 480 cmd+=' || ${diff_exe} ' 481 else: 482 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 483 cmd+=subst['label_suffix']+' ""' # Quotes are painful 484 cmdLines+=cmd+"\n" 485 cmdLines+=cmdindnt+'else\n' 486 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 487 cmdLines+=cmdindnt+'fi\n' 488 return cmdLines 489 490 def _writeTodoSkip(self,fh,tors,reasons,footer): 491 """ 492 Write out the TODO and SKIP lines in the file 493 The TODO or SKIP variable, tors, should be lower case 494 """ 495 TORS=tors.upper() 496 template=eval("example_template."+tors+"line") 497 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 498 tab = '' 499 if reasons: 500 fh.write('if ! $force; then\n') 501 tab = tab + ' ' 502 if reasons == ["Requires DATAFILESPATH"]: 503 # The only reason not to run is DATAFILESPATH, which we check at run-time 504 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 505 tab = tab + ' ' 506 if reasons: 507 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 508 fh.write(tab+footer+"\n") 509 fh.write(tab+"exit\n") 510 if reasons == ["Requires DATAFILESPATH"]: 511 fh.write(' fi\n') 512 if reasons: 513 fh.write('fi\n') 514 fh.write('\n\n') 515 return 516 517 def getLoopVarsHead(self,loopVars,i,usedVars={}): 518 """ 519 Generate a nicely indented string with the format loops 520 Here is what the data structure looks like 521 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 522 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 523 loopVars['subargs']['pc_type']=["j","cholesky sor"] 524 """ 525 outstr=''; indnt=self.indent 526 527 for key in loopVars: 528 for var in loopVars[key]['varlist']: 529 varval=loopVars[key][var] 530 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 531 outstr += "\n\n" 532 533 for key in loopVars: 534 for var in loopVars[key]['varlist']: 535 varval=loopVars[key][var] 536 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 537 i = i + 1 538 return (outstr,i) 539 540 def getLoopVarsFoot(self,loopVars,i): 541 outstr=''; indnt=self.indent 542 for key in loopVars: 543 for var in loopVars[key]['varlist']: 544 i = i - 1 545 outstr += indnt * i + "done\n" 546 return (outstr,i) 547 548 def genRunScript(self,testname,root,isRun,srcDict): 549 """ 550 Generate bash script using template found next to this file. 551 This file is read in at constructor time to avoid file I/O 552 """ 553 # runscript_dir directory has to be consistent with gmakefile 554 testDict=srcDict[testname] 555 rpath=self.srcrelpath(root) 556 runscript_dir=os.path.join(self.testroot_dir,rpath) 557 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 558 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 559 560 # Get variables to go into shell scripts. last time testDict used 561 subst=self.getSubstVars(testDict,rpath,testname) 562 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 563 if 'subtests' in testDict: 564 # The subtests inherit inDict, so we don't need top-level loops. 565 loopVars = {} 566 567 #Handle runfiles 568 for lfile in subst.get('localrunfiles','').split(): 569 install_files(os.path.join(root, lfile), 570 os.path.join(runscript_dir, os.path.dirname(lfile))) 571 # Check subtests for local runfiles 572 for stest in subst.get("subtests",[]): 573 for lfile in testDict[stest].get('localrunfiles','').split(): 574 install_files(os.path.join(root, lfile), 575 os.path.join(runscript_dir, os.path.dirname(lfile))) 576 577 # Now substitute the key variables into the header and footer 578 header=self._substVars(subst,example_template.header) 579 # The header is done twice to enable @...@ in header 580 header=self._substVars(subst,header) 581 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 582 583 # Start writing the file 584 fh.write(header+"\n") 585 586 # If there is a TODO or a SKIP then we do it before writing out the 587 # rest of the command (which is useful for working on the test) 588 # SKIP and TODO can be for the source file or for the runs 589 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 590 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 591 592 j=0 # for indentation 593 594 if loopVars: 595 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 596 if (loopHead): fh.write(loopHead+"\n") 597 598 # Subtests are special 599 allLoopVars=list(loopVars.keys()) 600 if 'subtests' in testDict: 601 substP=subst # Subtests can inherit args but be careful 602 k=0 # for label suffixes 603 for stest in testDict["subtests"]: 604 subst=substP.copy() 605 subst.update(testDict[stest]) 606 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 607 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 608 if sLoopVars: 609 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 610 allLoopVars+=list(sLoopVars.keys()) 611 fh.write(sLoopHead+"\n") 612 fh.write(self.getCmds(subst,j)+"\n") 613 if sLoopVars: 614 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 615 fh.write(sLoopFoot+"\n") 616 else: 617 fh.write(self.getCmds(subst,j)+"\n") 618 619 if loopVars: 620 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 621 fh.write(loopFoot+"\n") 622 623 fh.write(footer+"\n") 624 625 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 626 #if '10_9' in testname: sys.exit() 627 return 628 629 def genScriptsAndInfo(self,exfile,root,srcDict): 630 """ 631 Generate scripts from the source file, determine if built, etc. 632 For every test in the exfile with info in the srcDict: 633 1. Determine if it needs to be run for this arch 634 2. Generate the script 635 3. Generate the data needed to write out the makefile in a 636 convenient way 637 All tests are *always* run, but some may be SKIP'd per the TAP standard 638 """ 639 debug=False 640 rpath=self.srcrelpath(root) 641 execname=self.getExecname(exfile,rpath) 642 isBuilt=self._isBuilt(exfile,srcDict) 643 for test in srcDict: 644 if test in self.buildkeys: continue 645 if debug: print(nameSpace(exfile,root), test) 646 srcDict[test]['execname']=execname # Convenience in generating scripts 647 isRun=self._isRun(srcDict[test]) 648 self.genRunScript(test,root,isRun,srcDict) 649 srcDict[test]['isrun']=isRun 650 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 651 652 # This adds to datastructure for building deps 653 if isBuilt: self.addToSources(exfile,rpath,srcDict) 654 return 655 656 def _isBuilt(self,exfile,srcDict): 657 """ 658 Determine if this file should be built. 659 """ 660 # Get the language based on file extension 661 srcDict['SKIP'] = [] 662 lang=self.getLanguage(exfile) 663 if (lang=="F" or lang=="F90"): 664 if not self.have_fortran: 665 srcDict["SKIP"].append("Fortran required for this test") 666 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 667 srcDict["SKIP"].append("Fortran f90freeform required for this test") 668 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 669 srcDict["SKIP"].append("CUDA required for this test") 670 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 671 srcDict["SKIP"].append("HIP required for this test") 672 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 673 srcDict["SKIP"].append("SYCL required for this test") 674 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 675 srcDict["SKIP"].append("KOKKOS required for this test") 676 if lang=="raja_cxx" and 'PETSC_HAVE_RAJA' not in self.conf: 677 srcDict["SKIP"].append("RAJA required for this test") 678 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 679 srcDict["SKIP"].append("C++ required for this test") 680 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 681 srcDict["SKIP"].append("C++ required for this test") 682 683 # Deprecated source files 684 if srcDict.get("TODO"): 685 return False 686 687 # isRun can work with srcDict to handle the requires 688 if "requires" in srcDict: 689 if srcDict["requires"]: 690 return self._isRun(srcDict) 691 692 return srcDict['SKIP'] == [] 693 694 695 def _isRun(self,testDict, debug=False): 696 """ 697 Based on the requirements listed in the src file and the petscconf.h 698 info, determine whether this test should be run or not. 699 """ 700 indent=" " 701 702 if 'SKIP' not in testDict: 703 testDict['SKIP'] = [] 704 # MPI requirements 705 if 'MPI_IS_MPIUNI' in self.conf: 706 if testDict.get('nsize', '1') != '1': 707 testDict['SKIP'].append("Parallel test with serial build") 708 709 # The requirements for the test are the sum of all the run subtests 710 if 'subtests' in testDict: 711 if 'requires' not in testDict: testDict['requires']="" 712 for stest in testDict['subtests']: 713 if 'requires' in testDict[stest]: 714 testDict['requires']+=" "+testDict[stest]['requires'] 715 if testDict[stest].get('nsize', '1') != '1': 716 testDict['SKIP'].append("Parallel test with serial build") 717 break 718 719 # Now go through all requirements 720 if 'requires' in testDict: 721 for requirement in testDict['requires'].split(): 722 requirement=requirement.strip() 723 if not requirement: continue 724 if debug: print(indent+"Requirement: ", requirement) 725 isNull=False 726 if requirement.startswith("!"): 727 requirement=requirement[1:]; isNull=True 728 # Precision requirement for reals 729 if requirement in self.precision_types: 730 if self.conf['PETSC_PRECISION']==requirement: 731 if isNull: 732 testDict['SKIP'].append("not "+requirement+" required") 733 continue 734 continue # Success 735 elif not isNull: 736 testDict['SKIP'].append(requirement+" required") 737 continue 738 # Precision requirement for ints 739 if requirement in self.integer_types: 740 if requirement=="int32": 741 if self.conf['PETSC_SIZEOF_INT']==4: 742 if isNull: 743 testDict['SKIP'].append("not int32 required") 744 continue 745 continue # Success 746 elif not isNull: 747 testDict['SKIP'].append("int32 required") 748 continue 749 if requirement=="int64": 750 if self.conf['PETSC_SIZEOF_INT']==8: 751 if isNull: 752 testDict['SKIP'].append("NOT int64 required") 753 continue 754 continue # Success 755 elif not isNull: 756 testDict['SKIP'].append("int64 required") 757 continue 758 if requirement.startswith("long"): 759 reqsize = int(requirement[4:])//8 760 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 761 if longsize==reqsize: 762 if isNull: 763 testDict['SKIP'].append("not %s required" % requirement) 764 continue 765 continue # Success 766 elif not isNull: 767 testDict['SKIP'].append("%s required" % requirement) 768 continue 769 # Datafilespath 770 if requirement=="datafilespath" and not isNull: 771 testDict['SKIP'].append("Requires DATAFILESPATH") 772 continue 773 # Defines -- not sure I have comments matching 774 if "defined(" in requirement.lower(): 775 reqdef=requirement.split("(")[1].split(")")[0] 776 if reqdef in self.conf: 777 if isNull: 778 testDict['SKIP'].append("Null requirement not met: "+requirement) 779 continue 780 continue # Success 781 elif not isNull: 782 testDict['SKIP'].append("Required: "+requirement) 783 continue 784 785 # Rest should be packages that we can just get from conf 786 if requirement in ["complex","debug"]: 787 petscconfvar="PETSC_USE_"+requirement.upper() 788 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 789 else: 790 petscconfvar="PETSC_HAVE_"+requirement.upper() 791 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 792 petsccv = self.conf.get(petscconfvar) 793 pkgcv = self.conf.get(pkgconfvar) 794 795 if petsccv or pkgcv: 796 if isNull: 797 if petsccv: 798 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 799 continue 800 else: 801 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 802 continue 803 continue # Success 804 elif not isNull: 805 if not petsccv and not pkgcv: 806 if debug: print("requirement not found: ", requirement) 807 if self.pkg_name == 'petsc': 808 testDict['SKIP'].append(petscconfvar+" requirement not met") 809 else: 810 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 811 continue 812 return testDict['SKIP'] == [] 813 814 def checkOutput(self,exfile,root,srcDict): 815 """ 816 Check and make sure the output files are in the output directory 817 """ 818 debug=False 819 rpath=self.srcrelpath(root) 820 for test in srcDict: 821 if test in self.buildkeys: continue 822 if debug: print(rpath, exfile, test) 823 if 'output_file' in srcDict[test]: 824 output_file=srcDict[test]['output_file'] 825 else: 826 defroot = testparse.getDefaultOutputFileRoot(test) 827 if 'TODO' in srcDict[test]: continue 828 output_file="output/"+defroot+".out" 829 830 fullout=os.path.join(root,output_file) 831 if debug: print("---> ",fullout) 832 if not os.path.exists(fullout): 833 self.missing_files.append(fullout) 834 835 return 836 837 def genPetscTests_summarize(self,dataDict): 838 """ 839 Required method to state what happened 840 """ 841 if not self.summarize: return 842 indent=" " 843 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 844 with open(fhname, "w") as fh: 845 for root in dataDict: 846 relroot=self.srcrelpath(root) 847 pkg=relroot.split("/")[1] 848 fh.write(relroot+"\n") 849 allSrcs=[] 850 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 851 for exfile in dataDict[root]: 852 # Basic information 853 rfile=os.path.join(relroot,exfile) 854 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 855 fh.write(indent+exfile+indent*4+builtStatus+"\n") 856 for test in dataDict[root][exfile]: 857 if test in self.buildkeys: continue 858 line=indent*2+test 859 fh.write(line+"\n") 860 # Looks nice to have the keys in order 861 #for key in dataDict[root][exfile][test]: 862 for key in "isrun abstracted nsize args requires script".split(): 863 if key not in dataDict[root][exfile][test]: continue 864 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 865 fh.write(line+"\n") 866 fh.write("\n") 867 fh.write("\n") 868 fh.write("\n") 869 return 870 871 def genPetscTests(self,root,dirs,files,dataDict): 872 """ 873 Go through and parse the source files in the directory to generate 874 the examples based on the metadata contained in the source files 875 """ 876 debug=False 877 # Use examplesAnalyze to get what the makefles think are sources 878 #self.examplesAnalyze(root,dirs,files,anlzDict) 879 880 dataDict[root]={} 881 882 for exfile in files: 883 #TST: Until we replace files, still leaving the orginals as is 884 #if not exfile.startswith("new_"+"ex"): continue 885 #if not exfile.startswith("ex"): continue 886 887 # Ignore emacs and other temporary files 888 if exfile.startswith("."): continue 889 if exfile.startswith("#"): continue 890 if exfile.endswith("~"): continue 891 # Only parse source files 892 ext=getlangext(exfile).lstrip('.').replace('.','_') 893 if ext not in LANGS: continue 894 895 # Convenience 896 fullex=os.path.join(root,exfile) 897 if self.verbose: print(' --> '+fullex) 898 dataDict[root].update(testparse.parseTestFile(fullex,0)) 899 if exfile in dataDict[root]: 900 if not self.check_output: 901 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 902 else: 903 self.checkOutput(exfile,root,dataDict[root][exfile]) 904 905 return 906 907 def walktree(self,top): 908 """ 909 Walk a directory tree, starting from 'top' 910 """ 911 if self.check_output: 912 print("Checking for missing output files") 913 self.missing_files=[] 914 915 # Goal of action is to fill this dictionary 916 dataDict={} 917 for root, dirs, files in os.walk(top, topdown=True): 918 dirs.sort() 919 files.sort() 920 if "/tests" not in root and "/tutorials" not in root: continue 921 if "dSYM" in root: continue 922 if "tutorials"+os.sep+"build" in root: continue 923 if os.path.basename(root.rstrip("/")) == 'output': continue 924 if self.verbose: print(root) 925 self.genPetscTests(root,dirs,files,dataDict) 926 927 # If checking output, report results 928 if self.check_output: 929 if self.missing_files: 930 for file in set(self.missing_files): # set uniqifies 931 print(file) 932 sys.exit(1) 933 934 # Now summarize this dictionary 935 if self.verbose: self.genPetscTests_summarize(dataDict) 936 return dataDict 937 938 def gen_gnumake(self, fd): 939 """ 940 Overwrite of the method in the base PETSc class 941 """ 942 def write(stem, srcs): 943 for lang in LANGS: 944 if srcs[lang]['srcs']: 945 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 946 for pkg in self.pkg_pkgs: 947 srcs = self.gen_pkg(pkg) 948 write('testsrcs-' + pkg, srcs) 949 # Handle dependencies 950 for lang in LANGS: 951 for exfile in srcs[lang]['srcs']: 952 if exfile in srcs[lang]: 953 ex='$(TESTDIR)/'+getlangsplit(exfile) 954 exfo=ex+'.o' 955 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 956 if deps: 957 # The executable literally depends on the object file because it is linked 958 fd.write(ex +": " + " ".join(deps) +'\n') 959 # The object file containing 'main' does not normally depend on other object 960 # files, but it does when it includes their modules. This dependency is 961 # overly blunt and could be reduced to only depend on object files for 962 # modules that are used, like "*f90aux.o". 963 fd.write(exfo +": " + " ".join(deps) +'\n') 964 965 return self.gendeps 966 967 def gen_pkg(self, pkg): 968 """ 969 Overwrite of the method in the base PETSc class 970 """ 971 return self.sources[pkg] 972 973 def write_gnumake(self, dataDict, output=None): 974 """ 975 Write out something similar to files from gmakegen.py 976 977 Test depends on script which also depends on source 978 file, but since I don't have a good way generating 979 acting on a single file (oops) just depend on 980 executable which in turn will depend on src file 981 """ 982 # Different options for how to set up the targets 983 compileExecsFirst=False 984 985 # Open file 986 with open(output, 'w') as fd: 987 # Write out the sources 988 gendeps = self.gen_gnumake(fd) 989 990 # Write out the tests and execname targets 991 fd.write("\n#Tests and executables\n") # Delimiter 992 993 for pkg in self.pkg_pkgs: 994 # These grab the ones that are built 995 for lang in LANGS: 996 testdeps=[] 997 for ftest in self.tests[pkg][lang]: 998 test=os.path.basename(ftest) 999 basedir=os.path.dirname(ftest) 1000 testdeps.append(nameSpace(test,basedir)) 1001 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 1002 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 1003 1004 # test targets 1005 for ftest in self.tests[pkg][lang]: 1006 test=os.path.basename(ftest) 1007 basedir=os.path.dirname(ftest) 1008 testdir="${TESTDIR}/"+basedir+"/" 1009 nmtest=nameSpace(test,basedir) 1010 rundir=os.path.join(testdir,test) 1011 script=test+".sh" 1012 1013 # Deps 1014 exfile=self.tests[pkg][lang][ftest]['exfile'] 1015 fullex=os.path.join(self.srcdir,exfile) 1016 localexec=self.tests[pkg][lang][ftest]['exec'] 1017 execname=os.path.join(testdir,localexec) 1018 fullscript=os.path.join(testdir,script) 1019 tmpfile=os.path.join(testdir,test,test+".tmp") 1020 1021 # *.counts depends on the script and either executable (will 1022 # be run) or the example source file (SKIP or TODO) 1023 fd.write('%s.counts : %s %s' 1024 % (os.path.join('$(TESTDIR)/counts', nmtest), 1025 fullscript, 1026 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1027 ) 1028 if exfile in self.sources[pkg][lang]: 1029 for dep in self.sources[pkg][lang][exfile]: 1030 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1031 fd.write('\n') 1032 1033 # Now write the args: 1034 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1035 1036 return 1037 1038 def write_db(self, dataDict, testdir): 1039 """ 1040 Write out the dataDict into a pickle file 1041 """ 1042 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1043 pickle.dump(dataDict,fd) 1044 return 1045 1046def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1047 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1048 srcdir=None, testdir=None, check=False): 1049 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1050 testdir=os.path.normpath(testdir) 1051 if petsc_arch: 1052 petsc_arch=petsc_arch.rstrip(os.path.sep) 1053 if len(petsc_arch.split(os.path.sep))>1: 1054 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1055 output = os.path.join(testdir, 'testfiles') 1056 1057 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1058 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1059 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1060 testdir=testdir,check=check) 1061 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1062 if not pEx.check_output: 1063 pEx.write_gnumake(dataDict, output) 1064 pEx.write_db(dataDict, testdir) 1065 1066if __name__ == '__main__': 1067 import optparse 1068 parser = optparse.OptionParser() 1069 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1070 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1071 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1072 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1073 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1074 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1075 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1076 help='Check whether output files are in output director') 1077 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1078 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1079 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1080 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1081 1082 opts, extra_args = parser.parse_args() 1083 if extra_args: 1084 import sys 1085 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1086 exit(1) 1087 if opts.testdir is None: 1088 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1089 1090 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1091 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1092 verbose=opts.verbose, 1093 single_ex=opts.single_executable, srcdir=opts.srcdir, 1094 testdir=opts.testdir, check=opts.check_output) 1095