1#!/usr/bin/env python 2import fnmatch 3import glob 4import inspect 5import os 6import optparse 7import pickle 8import re 9import sys 10 11thisfile = os.path.abspath(inspect.getfile(inspect.currentframe())) 12pdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(thisfile))))) 13sys.path.insert(0, os.path.join(pdir, 'config')) 14 15import testparse 16from gmakegentest import nameSpace 17 18 19""" 20 Tool for querying the tests. 21 22 Which tests to query? Two options: 23 1. Query only the tests that are run for a given configuration. 24 2. Query all of the test files in the source directory 25 For #1: 26 Use dataDict as written out by gmakegentest.py in $PETSC_ARCH/$TESTBASE 27 For #2: 28 Walk the entire tree parsing the files as we go along using testparse. 29 The tree walker is simpler than what is in gmakegentest.py 30 31 The dataDict follows that generated by testparse. gmakegentest.py does 32 further manipulations of the dataDict to handle things like for loops 33 so if using #2, those modifications are not included. 34 35 Querying: 36 The dataDict dictionary is then "inverted" to create a dictionary with the 37 range of field values as keys and list test names as the values. This 38 allows fast searching 39 40""" 41 42def isFile(maybeFile): 43 ext=os.path.splitext(maybeFile)[1] 44 if not ext: return False 45 if ext not in ['.c','.cxx','.cpp','F90','F','cu']: return False 46 return True 47 48def pathToLabel(path): 49 """ 50 Because the scripts have a non-unique naming, the pretty-printing 51 needs to convey the srcdir and srcfile. There are two ways of doing this. 52 """ 53 # Strip off any top-level directories or spaces 54 path=path.strip().replace(pdir,'') 55 path=path.replace('src/','') 56 if isFile(path): 57 prefix=os.path.dirname(path).replace("/","_") 58 suffix=os.path.splitext(os.path.basename(path))[0] 59 label=prefix+"-"+suffix+'_*' 60 else: 61 path=path.rstrip('/') 62 label=path.replace("/","_")+"-*" 63 return label 64 65def get_value(varset): 66 """ 67 Searching args is a bit funky: 68 Consider 69 args: -ksp_monitor_short -pc_type ml -ksp_max_it 3 70 Search terms are: 71 ksp_monitor, 'pc_type ml', ksp_max_it 72 Also ignore all loops 73 -pc_fieldsplit_diag_use_amat {{0 1}} 74 Gives: pc_fieldsplit_diag_use_amat as the search term 75 Also ignore -f ... (use matrices from file) because I'll assume 76 that this kind of information isn't needed for testing. If it's 77 a separate search than just grep it 78 """ 79 if varset.startswith('-f '): return None 80 81 # First remove loops 82 value=re.sub('{{.*}}','',varset) 83 # Next remove - 84 value=varset.lstrip("-") 85 # Get rid of numbers 86 value=re.sub(r"[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?",'',value) 87 # return without spaces 88 return value.strip() 89 90def query(invDict,fields,labels): 91 """ 92 Search the keys using fnmatch to find matching names and return list with 93 the results 94 """ 95 setlist=[] # setlist is a list of lists that set opertions will operate on 96 llist=labels.replace('|',',').split(',') 97 i=-1 98 for field in fields.replace('|',',').split(','): 99 i+=1 100 label=llist[i] 101 if field == 'name': 102 if '/' in label: 103 label=pathToLabel(label) 104 setlist.append(fnmatch.filter(invDict['name'],label)) 105 continue 106 107 foundLabel=False # easy to do if you misspell argument search 108 for key in invDict[field]: 109 if fnmatch.filter([key],label): 110 foundLabel=True 111 # Do not return values with not unless label itself has not 112 if label.startswith('!') and not key.startswith('!'): continue 113 if not label.startswith('!') and key.startswith('!'): continue 114 setlist.append(invDict[field][key]) 115 if not foundLabel: 116 setlist.append([]) 117 118 # Now process the union and intersection operators based on setlist 119 allresults=[] 120 # Union 121 i=-1 122 for ufield in fields.split(','): 123 i+=1 124 if '|' in ufield: 125 # Intersection 126 label=llist[i] 127 results=set(setlist[i]) 128 for field in ufield.split('|')[1:]: 129 i+=1 130 label=llist[i] 131 results=results.intersection(set(setlist[i])) 132 allresults+=list(results) 133 else: 134 allresults+=setlist[i] 135 136 # remove duplicate entries and sort to give consistent results 137 uniqlist=list(set(allresults)) 138 uniqlist.sort() 139 return uniqlist 140 141def get_inverse_dictionary(dataDict,fields,srcdir): 142 """ 143 Create a dictionary with the values of field as the keys, and the name of 144 the tests as the results. 145 """ 146 invDict={} 147 # Comma-delimited lists denote union 148 for field in fields.replace('|',',').split(','): 149 if field not in invDict: 150 if field == 'name': 151 invDict[field]=[] # List for ease 152 else: 153 invDict[field]={} 154 for root in dataDict: 155 for exfile in dataDict[root]: 156 for test in dataDict[root][exfile]: 157 if test in testparse.buildkeys: continue 158 defroot = testparse.getDefaultOutputFileRoot(test) 159 fname=nameSpace(defroot,os.path.relpath(root,srcdir)) 160 if field == 'name': 161 invDict['name'].append(fname) 162 continue 163 if field not in dataDict[root][exfile][test]: continue 164 values=dataDict[root][exfile][test][field] 165 166 if not field == 'args' and not field == 'diff_args': 167 for val in values.split(): 168 if val in invDict[field]: 169 invDict[field][val].append(fname) 170 else: 171 invDict[field][val] = [fname] 172 else: 173 # Args are funky. 174 for varset in re.split('(^|\W)-(?=[a-zA-Z])',values): 175 val=get_value(varset) 176 if not val: continue 177 if val in invDict[field]: 178 invDict[field][val].append(fname) 179 else: 180 invDict[field][val] = [fname] 181 # remove duplicate entries (multiple test/file) 182 if not field == 'name': 183 for val in invDict[field]: 184 invDict[field][val]=list(set(invDict[field][val])) 185 186 return invDict 187 188def get_gmakegentest_data(testdir,petsc_dir,petsc_arch): 189 """ 190 Write out the dataDict into a pickle file 191 """ 192 # This needs to be consistent with gmakegentest.py of course 193 pkl_file=os.path.join(testdir,'datatest.pkl') 194 # If it doesn't exist, then we need to regenerate 195 if not os.path.exists(pkl_file): 196 startdir=os.path.abspath(os.curdir) 197 os.chdir(petsc_dir) 198 args='--petsc-dir='+petsc_dir+' --petsc-arch='+petsc_arch+' --testdir='+testdir 199 buf = os.popen('config/gmakegentest.py '+args).read() 200 os.chdir(startdir) 201 202 fd = open(pkl_file, 'rb') 203 dataDict=pickle.load(fd) 204 fd.close() 205 return dataDict 206 207def walktree(top): 208 """ 209 Walk a directory tree, starting from 'top' 210 """ 211 verbose = False 212 dataDict = {} 213 alldatafiles = [] 214 for root, dirs, files in os.walk(top, topdown=False): 215 if root == 'output': continue 216 if '.dSYM' in root: continue 217 if verbose: print(root) 218 219 dataDict[root] = {} 220 221 for exfile in files: 222 # Ignore emacs files 223 if exfile.startswith("#") or exfile.startswith(".#"): continue 224 ext=os.path.splitext(exfile)[1] 225 if ext[1:] not in ['c','cxx','cpp','cu','F90','F']: continue 226 227 # Convenience 228 fullex = os.path.join(root, exfile) 229 if verbose: print(' --> '+fullex) 230 dataDict[root].update(testparse.parseTestFile(fullex, 0)) 231 232 return dataDict 233 234def do_query(use_source, startdir, srcdir, testdir, petsc_dir, petsc_arch, 235 fields, labels, searchin): 236 """ 237 Do the actual query 238 This part of the code is placed here instead of main() 239 to show how one could translate this into ipython/jupyer notebook 240 commands for more advanced queries 241 """ 242 # Get dictionary 243 if use_source: 244 dataDict=walktree(startdir) 245 else: 246 dataDict=get_gmakegentest_data(testdir, petsc_dir, petsc_arch) 247 248 # Get inverse dictionary for searching 249 invDict=get_inverse_dictionary(dataDict, fields, srcdir) 250 251 # Now do query 252 resList=query(invDict, fields, labels) 253 254 # Filter results using searchin 255 newresList=[] 256 if searchin.strip(): 257 for key in resList: 258 if fnmatch.filter([key],searchin): 259 newresList.append(key) 260 resList=newresList 261 262 # Print in flat list suitable for use by gmakefile.test 263 print(' '.join(resList)) 264 265 return 266 267def main(): 268 parser = optparse.OptionParser(usage="%prog [options] field match_pattern") 269 parser.add_option('-s', '--startdir', dest='startdir', 270 help='Where to start the recursion if not srcdir', 271 default='') 272 parser.add_option('-p', '--petsc-dir', dest='petsc_dir', 273 help='Set PETSC_DIR different from environment', 274 default=os.environ.get('PETSC_DIR')) 275 parser.add_option('-a', '--petsc-arch', dest='petsc_arch', 276 help='Set PETSC_ARCH different from environment', 277 default=os.environ.get('PETSC_ARCH')) 278 parser.add_option('--srcdir', dest='srcdir', 279 help='Set location of sources different from PETSC_DIR/src. Must be full path.', 280 default='src') 281 parser.add_option('-t', '--testdir', dest='testdir', 282 help='Test directory if not PETSC_ARCH/tests. Must be full path', 283 default='tests') 284 parser.add_option('-u', '--use-source', action="store_false", 285 dest='use_source', 286 help='Query all sources rather than those configured in PETSC_ARCH') 287 parser.add_option('-i', '--searchin', dest='searchin', 288 help='Filter results from the arguments', 289 default='') 290 291 opts, args = parser.parse_args() 292 293 # Argument Sanity checks 294 if len(args) != 2: 295 parser.print_usage() 296 print('Arguments: ') 297 print(' field: Field to search for; e.g., requires') 298 print(' To just match names, use "name"') 299 print(' match_pattern: Matching pattern for field; e.g., cuda') 300 return 301 302 # Process arguments and options -- mostly just paths here 303 field=args[0] 304 match=args[1] 305 searchin=opts.searchin 306 307 petsc_dir = opts.petsc_dir 308 petsc_arch = opts.petsc_arch 309 petsc_full_arch = os.path.join(petsc_dir, petsc_arch) 310 311 if opts.srcdir == 'src': 312 petsc_full_src = os.path.join(petsc_dir, 'src') 313 else: 314 petsc_full_src = opts.srcdir 315 if opts.testdir == 'tests': 316 petsc_full_test = os.path.join(petsc_full_arch, 'tests') 317 else: 318 petsc_full_test = opts.testdir 319 if opts.startdir: 320 startdir=opts.startdir=petsc_full_src 321 else: 322 startdir=petsc_full_src 323 324 # Options Sanity checks 325 if not os.path.isdir(petsc_dir): 326 print("PETSC_DIR must be a directory") 327 return 328 329 if not opts.use_source: 330 if not os.path.isdir(petsc_full_arch): 331 print("PETSC_DIR/PETSC_ARCH must be a directory") 332 return 333 elif not os.path.isdir(petsc_full_test): 334 print("Testdir must be a directory"+petsc_full_test) 335 return 336 else: 337 if not os.path.isdir(petsc_full_src): 338 print("Source directory must be a directory"+petsc_full_src) 339 return 340 341 # Do the actual query 342 do_query(opts.use_source, startdir, petsc_full_src, petsc_full_test, 343 petsc_dir, petsc_arch, field, match, searchin) 344 345 return 346 347 348if __name__ == "__main__": 349 main() 350