1#!/usr/bin/env python 2import fnmatch 3import glob 4import inspect 5import os 6import optparse 7import pickle 8import re 9import sys 10 11thisfile = os.path.abspath(inspect.getfile(inspect.currentframe())) 12pdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(thisfile))))) 13sys.path.insert(0, os.path.join(pdir, 'config')) 14 15import testparse 16from gmakegentest import nameSpace 17 18 19""" 20 Tool for querying the tests. 21 22 Which tests to query? Two options: 23 1. Query only the tests that are run for a given configuration. 24 2. Query all of the test files in the source directory 25 For #1: 26 Use dataDict as written out by gmakegentest.py in $PETSC_ARCH/$TESTBASE 27 For #2: 28 Walk the entire tree parsing the files as we go along using testparse. 29 The tree walker is simpler than what is in gmakegentest.py 30 31 The dataDict follows that generated by testparse. gmakegentest.py does 32 further manipulations of the dataDict to handle things like for loops 33 so if using #2, those modifications are not included. 34 35 Querying: 36 The dataDict dictionary is then "inverted" to create a dictionary with the 37 range of field values as keys and list test names as the values. This 38 allows fast searching 39 40""" 41 42def isFile(maybeFile): 43 ext=os.path.splitext(maybeFile)[1] 44 if not ext: return False 45 if ext not in ['.c','.cxx','.cpp','F90','F','cu']: return False 46 return True 47 48def pathToLabel(path): 49 """ 50 Because the scripts have a non-unique naming, the pretty-printing 51 needs to convey the srcdir and srcfile. There are two ways of doing this. 52 """ 53 # Strip off any top-level directories or spaces 54 path=path.strip().replace(pdir,'') 55 path=path.replace('src/','') 56 if isFile(path): 57 prefix=os.path.dirname(path).replace("/","_") 58 suffix=os.path.splitext(os.path.basename(path))[0] 59 label=prefix+"-"+suffix+'_*' 60 else: 61 path=path.rstrip('/') 62 label=path.replace("/","_")+"-*" 63 return label 64 65def get_value(varset): 66 """ 67 Searching args is a bit funky: 68 Consider 69 args: -ksp_monitor_short -pc_type ml -ksp_max_it 3 70 Search terms are: 71 ksp_monitor, 'pc_type ml', ksp_max_it 72 Also ignore all loops 73 -pc_fieldsplit_diag_use_amat {{0 1}} 74 Gives: pc_fieldsplit_diag_use_amat as the search term 75 Also ignore -f ... (use matrices from file) because I'll assume 76 that this kind of information isn't needed for testing. If it's 77 a separate search than just grep it 78 """ 79 if varset.startswith('-f '): return None 80 81 # First remove loops 82 value=re.sub('{{.*}}','',varset) 83 # Next remove - 84 value=varset.lstrip("-") 85 # Get rid of numbers 86 value=re.sub(r"[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?",'',value) 87 # return without spaces 88 return value.strip() 89 90def query(invDict,fields,labels): 91 """ 92 Search the keys using fnmatch to find matching names and return list with 93 the results 94 """ 95 setlist=[] # setlist is a list of lists that set opertions will operate on 96 llist=labels.replace('|',',').split(',') 97 i=-1 98 for field in fields.replace('|',',').split(','): 99 i+=1 100 label=llist[i] 101 if field == 'name': 102 if '/' in label: 103 label=pathToLabel(label) 104 elif label.startswith('src'): 105 label=label.lstrip('src').lstrip('*') 106 setlist.append(fnmatch.filter(invDict['name'],label)) 107 continue 108 109 foundLabel=False # easy to do if you misspell argument search 110 for key in invDict[field]: 111 if fnmatch.filter([key],label): 112 foundLabel=True 113 # Do not return values with not unless label itself has not 114 if label.startswith('!') and not key.startswith('!'): continue 115 if not label.startswith('!') and key.startswith('!'): continue 116 setlist.append(invDict[field][key]) 117 if not foundLabel: 118 setlist.append([]) 119 120 # Now process the union and intersection operators based on setlist 121 allresults=[] 122 # Union 123 i=-1 124 for ufield in fields.split(','): 125 i+=1 126 if '|' in ufield: 127 # Intersection 128 label=llist[i] 129 results=set(setlist[i]) 130 for field in ufield.split('|')[1:]: 131 i+=1 132 label=llist[i] 133 results=results.intersection(set(setlist[i])) 134 allresults+=list(results) 135 else: 136 allresults+=setlist[i] 137 138 # remove duplicate entries and sort to give consistent results 139 uniqlist=list(set(allresults)) 140 uniqlist.sort() 141 return uniqlist 142 143def get_inverse_dictionary(dataDict,fields,srcdir): 144 """ 145 Create a dictionary with the values of field as the keys, and the name of 146 the tests as the results. 147 """ 148 invDict={} 149 # Comma-delimited lists denote union 150 for field in fields.replace('|',',').split(','): 151 if field not in invDict: 152 if field == 'name': 153 invDict[field]=[] # List for ease 154 else: 155 invDict[field]={} 156 for root in dataDict: 157 for exfile in dataDict[root]: 158 for test in dataDict[root][exfile]: 159 if test in testparse.buildkeys: continue 160 defroot = testparse.getDefaultOutputFileRoot(test) 161 fname=nameSpace(defroot,os.path.relpath(root,srcdir)) 162 if field == 'name': 163 invDict['name'].append(fname) 164 continue 165 if field not in dataDict[root][exfile][test]: continue 166 values=dataDict[root][exfile][test][field] 167 168 if not field == 'args' and not field == 'diff_args': 169 for val in values.split(): 170 if val in invDict[field]: 171 invDict[field][val].append(fname) 172 else: 173 invDict[field][val] = [fname] 174 else: 175 # Args are funky. 176 for varset in re.split('(^|\W)-(?=[a-zA-Z])',values): 177 val=get_value(varset) 178 if not val: continue 179 if val in invDict[field]: 180 invDict[field][val].append(fname) 181 else: 182 invDict[field][val] = [fname] 183 # remove duplicate entries (multiple test/file) 184 if not field == 'name': 185 for val in invDict[field]: 186 invDict[field][val]=list(set(invDict[field][val])) 187 188 return invDict 189 190def get_gmakegentest_data(testdir,petsc_dir,petsc_arch): 191 """ 192 Write out the dataDict into a pickle file 193 """ 194 # This needs to be consistent with gmakegentest.py of course 195 pkl_file=os.path.join(testdir,'datatest.pkl') 196 # If it doesn't exist, then we need to regenerate 197 if not os.path.exists(pkl_file): 198 startdir=os.path.abspath(os.curdir) 199 os.chdir(petsc_dir) 200 args='--petsc-dir='+petsc_dir+' --petsc-arch='+petsc_arch+' --testdir='+testdir 201 buf = os.popen('config/gmakegentest.py '+args).read() 202 os.chdir(startdir) 203 204 fd = open(pkl_file, 'rb') 205 dataDict=pickle.load(fd) 206 fd.close() 207 return dataDict 208 209def walktree(top): 210 """ 211 Walk a directory tree, starting from 'top' 212 """ 213 verbose = False 214 dataDict = {} 215 alldatafiles = [] 216 for root, dirs, files in os.walk(top, topdown=False): 217 if root == 'output': continue 218 if '.dSYM' in root: continue 219 if verbose: print(root) 220 221 dataDict[root] = {} 222 223 for exfile in files: 224 # Ignore emacs files 225 if exfile.startswith("#") or exfile.startswith(".#"): continue 226 ext=os.path.splitext(exfile)[1] 227 if ext[1:] not in ['c','cxx','cpp','cu','F90','F']: continue 228 229 # Convenience 230 fullex = os.path.join(root, exfile) 231 if verbose: print(' --> '+fullex) 232 dataDict[root].update(testparse.parseTestFile(fullex, 0)) 233 234 return dataDict 235 236def do_query(use_source, startdir, srcdir, testdir, petsc_dir, petsc_arch, 237 fields, labels, searchin): 238 """ 239 Do the actual query 240 This part of the code is placed here instead of main() 241 to show how one could translate this into ipython/jupyer notebook 242 commands for more advanced queries 243 """ 244 # Get dictionary 245 if use_source: 246 dataDict=walktree(startdir) 247 else: 248 dataDict=get_gmakegentest_data(testdir, petsc_dir, petsc_arch) 249 250 # Get inverse dictionary for searching 251 invDict=get_inverse_dictionary(dataDict, fields, srcdir) 252 253 # Now do query 254 resList=query(invDict, fields, labels) 255 256 # Filter results using searchin 257 newresList=[] 258 if searchin.strip(): 259 if not searchin.startswith('!'): 260 for key in resList: 261 if fnmatch.filter([key],searchin): 262 newresList.append(key) 263 else: 264 for key in resList: 265 if not fnmatch.filter([key],searchin[1:]): 266 newresList.append(key) 267 resList=newresList 268 269 # Print in flat list suitable for use by gmakefile.test 270 print(' '.join(resList)) 271 272 return 273 274def main(): 275 parser = optparse.OptionParser(usage="%prog [options] field match_pattern") 276 parser.add_option('-s', '--startdir', dest='startdir', 277 help='Where to start the recursion if not srcdir', 278 default='') 279 parser.add_option('-p', '--petsc-dir', dest='petsc_dir', 280 help='Set PETSC_DIR different from environment', 281 default=os.environ.get('PETSC_DIR')) 282 parser.add_option('-a', '--petsc-arch', dest='petsc_arch', 283 help='Set PETSC_ARCH different from environment', 284 default=os.environ.get('PETSC_ARCH')) 285 parser.add_option('--srcdir', dest='srcdir', 286 help='Set location of sources different from PETSC_DIR/src. Must be full path.', 287 default='src') 288 parser.add_option('-t', '--testdir', dest='testdir', 289 help='Test directory if not PETSC_ARCH/tests. Must be full path', 290 default='tests') 291 parser.add_option('-u', '--use-source', action="store_false", 292 dest='use_source', 293 help='Query all sources rather than those configured in PETSC_ARCH') 294 parser.add_option('-i', '--searchin', dest='searchin', 295 help='Filter results from the arguments', 296 default='') 297 298 opts, args = parser.parse_args() 299 300 # Argument Sanity checks 301 if len(args) != 2: 302 parser.print_usage() 303 print('Arguments: ') 304 print(' field: Field to search for; e.g., requires') 305 print(' To just match names, use "name"') 306 print(' match_pattern: Matching pattern for field; e.g., cuda') 307 return 308 309 # Process arguments and options -- mostly just paths here 310 field=args[0] 311 match=args[1] 312 searchin=opts.searchin 313 314 petsc_dir = opts.petsc_dir 315 petsc_arch = opts.petsc_arch 316 petsc_full_arch = os.path.join(petsc_dir, petsc_arch) 317 318 if petsc_arch == '': 319 petsc_full_src = os.path.join(petsc_dir, 'share', 'petsc', 'examples', 'src') 320 else: 321 if opts.srcdir == 'src': 322 petsc_full_src = os.path.join(petsc_dir, 'src') 323 else: 324 petsc_full_src = opts.srcdir 325 if opts.testdir == 'tests': 326 petsc_full_test = os.path.join(petsc_full_arch, 'tests') 327 else: 328 petsc_full_test = opts.testdir 329 if opts.startdir: 330 startdir=opts.startdir=petsc_full_src 331 else: 332 startdir=petsc_full_src 333 334 # Options Sanity checks 335 if not os.path.isdir(petsc_dir): 336 print("PETSC_DIR must be a directory") 337 return 338 339 if not opts.use_source: 340 if not os.path.isdir(petsc_full_arch): 341 print("PETSC_DIR/PETSC_ARCH must be a directory") 342 return 343 elif not os.path.isdir(petsc_full_test): 344 print("Testdir must be a directory"+petsc_full_test) 345 return 346 else: 347 if not os.path.isdir(petsc_full_src): 348 print("Source directory must be a directory"+petsc_full_src) 349 return 350 351 # Do the actual query 352 do_query(opts.use_source, startdir, petsc_full_src, petsc_full_test, 353 petsc_dir, petsc_arch, field, match, searchin) 354 355 return 356 357 358if __name__ == "__main__": 359 main() 360