1#!/usr/bin/env python 2import fnmatch 3import glob 4import inspect 5import os 6import optparse 7import pickle 8import re 9import sys 10 11thisfile = os.path.abspath(inspect.getfile(inspect.currentframe())) 12pdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(thisfile))))) 13sys.path.insert(0, os.path.join(pdir, 'config')) 14 15import testparse 16from gmakegentest import nameSpace 17 18 19""" 20 Tool for querying the tests. 21 22 Which tests to query? Two options: 23 1. Query only the tests that are run for a given configuration. 24 2. Query all of the test files in the source directory 25 For #1: 26 Use dataDict as written out by gmakegentest.py in $PETSC_ARCH/$TESTBASE 27 For #2: 28 Walk the entire tree parsing the files as we go along using testparse. 29 The tree walker is simpler than what is in gmakegentest.py 30 31 The dataDict follows that generated by testparse. gmakegentest.py does 32 further manipulations of the dataDict to handle things like for loops 33 so if using #2, those modifications are not included. 34 35 Querying: 36 The dataDict dictionary is then "inverted" to create a dictionary with the 37 range of field values as keys and list test names as the values. This 38 allows fast searching 39 40""" 41 42def isFile(maybeFile): 43 ext=os.path.splitext(maybeFile)[1] 44 if not ext: return False 45 if ext not in ['.c','.cxx','.cpp','F90','F','cu']: return False 46 return True 47 48def pathToLabel(path): 49 """ 50 Because the scripts have a non-unique naming, the pretty-printing 51 needs to convey the srcdir and srcfile. There are two ways of doing this. 52 """ 53 # Strip off any top-level directories or spaces 54 path=path.strip().replace(pdir,'') 55 path=path.replace('src/','') 56 if isFile(path): 57 prefix=os.path.dirname(path).replace("/","_") 58 suffix=os.path.splitext(os.path.basename(path))[0] 59 label=prefix+"-"+suffix+'_*' 60 else: 61 path=path.rstrip('/') 62 label=path.replace("/","_")+"-*" 63 return label 64 65def get_value(varset): 66 """ 67 Searching args is a bit funky: 68 Consider 69 args: -ksp_monitor_short -pc_type ml -ksp_max_it 3 70 Search terms are: 71 ksp_monitor, 'pc_type ml', ksp_max_it 72 Also ignore all loops 73 -pc_fieldsplit_diag_use_amat {{0 1}} 74 Gives: pc_fieldsplit_diag_use_amat as the search term 75 Also ignore -f ... (use matrices from file) because I'll assume 76 that this kind of information isn't needed for testing. If it's 77 a separate search than just grep it 78 """ 79 if varset.startswith('-f '): return None 80 81 # First remove loops 82 value=re.sub('{{.*}}','',varset) 83 # Next remove - 84 value=varset.lstrip("-") 85 # Get rid of numbers 86 value=re.sub(r"[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?",'',value) 87 # return without spaces 88 return value.strip() 89 90def query(invDict,fields,labels): 91 """ 92 Search the keys using fnmatch to find matching names and return list with 93 the results 94 """ 95 setlist=[] # setlist is a list of lists that set opertions will operate on 96 llist=labels.replace('|',',').split(',') 97 i=-1 98 for field in fields.replace('|',',').split(','): 99 i+=1 100 label=llist[i] 101 if field == 'name': 102 if '/' in label: 103 label=pathToLabel(label) 104 elif label.startswith('src'): 105 label=label.lstrip('src').lstrip('*') 106 setlist.append(fnmatch.filter(invDict['name'],label)) 107 continue 108 109 foundLabel=False # easy to do if you misspell argument search 110 for key in invDict[field]: 111 if fnmatch.filter([key],label): 112 foundLabel=True 113 # Do not return values with not unless label itself has not 114 if label.startswith('!') and not key.startswith('!'): continue 115 if not label.startswith('!') and key.startswith('!'): continue 116 setlist.append(invDict[field][key]) 117 if not foundLabel: 118 setlist.append([]) 119 120 # Now process the union and intersection operators based on setlist 121 allresults=[] 122 # Union 123 i=-1 124 for ufield in fields.split(','): 125 i+=1 126 if '|' in ufield: 127 # Intersection 128 label=llist[i] 129 results=set(setlist[i]) 130 for field in ufield.split('|')[1:]: 131 i+=1 132 label=llist[i] 133 results=results.intersection(set(setlist[i])) 134 allresults+=list(results) 135 else: 136 allresults+=setlist[i] 137 138 # remove duplicate entries and sort to give consistent results 139 uniqlist=list(set(allresults)) 140 uniqlist.sort() 141 return uniqlist 142 143def get_inverse_dictionary(dataDict,fields,srcdir): 144 """ 145 Create a dictionary with the values of field as the keys, and the name of 146 the tests as the results. 147 """ 148 invDict={} 149 # Comma-delimited lists denote union 150 for field in fields.replace('|',',').split(','): 151 if field not in invDict: 152 if field == 'name': 153 invDict[field]=[] # List for ease 154 else: 155 invDict[field]={} 156 for root in dataDict: 157 for exfile in dataDict[root]: 158 for test in dataDict[root][exfile]: 159 if test in testparse.buildkeys: continue 160 defroot = testparse.getDefaultOutputFileRoot(test) 161 fname=nameSpace(defroot,os.path.relpath(root,srcdir)) 162 if field == 'name': 163 invDict['name'].append(fname) 164 continue 165 if field not in dataDict[root][exfile][test]: continue 166 values=dataDict[root][exfile][test][field] 167 168 if not field == 'args' and not field == 'diff_args': 169 for val in values.split(): 170 if val in invDict[field]: 171 invDict[field][val].append(fname) 172 else: 173 invDict[field][val] = [fname] 174 else: 175 # Args are funky. 176 for varset in re.split('(^|\W)-(?=[a-zA-Z])',values): 177 val=get_value(varset) 178 if not val: continue 179 if val in invDict[field]: 180 invDict[field][val].append(fname) 181 else: 182 invDict[field][val] = [fname] 183 # remove duplicate entries (multiple test/file) 184 if not field == 'name': 185 for val in invDict[field]: 186 invDict[field][val]=list(set(invDict[field][val])) 187 188 return invDict 189 190def get_gmakegentest_data(testdir,petsc_dir,petsc_arch): 191 """ 192 Write out the dataDict into a pickle file 193 """ 194 # This needs to be consistent with gmakegentest.py of course 195 pkl_file=os.path.join(testdir,'datatest.pkl') 196 # If it doesn't exist, then we need to regenerate 197 if not os.path.exists(pkl_file): 198 startdir=os.path.abspath(os.curdir) 199 os.chdir(petsc_dir) 200 args='--petsc-dir='+petsc_dir+' --petsc-arch='+petsc_arch+' --testdir='+testdir 201 buf = os.popen('config/gmakegentest.py '+args).read() 202 os.chdir(startdir) 203 204 fd = open(pkl_file, 'rb') 205 dataDict=pickle.load(fd) 206 fd.close() 207 return dataDict 208 209def walktree(top): 210 """ 211 Walk a directory tree, starting from 'top' 212 """ 213 verbose = False 214 dataDict = {} 215 alldatafiles = [] 216 for root, dirs, files in os.walk(top, topdown=False): 217 if root == 'output': continue 218 if '.dSYM' in root: continue 219 if verbose: print(root) 220 221 dataDict[root] = {} 222 223 for exfile in files: 224 # Ignore emacs files 225 if exfile.startswith("#") or exfile.startswith(".#"): continue 226 ext=os.path.splitext(exfile)[1] 227 if ext[1:] not in ['c','cxx','cpp','cu','F90','F']: continue 228 229 # Convenience 230 fullex = os.path.join(root, exfile) 231 if verbose: print(' --> '+fullex) 232 dataDict[root].update(testparse.parseTestFile(fullex, 0)) 233 234 return dataDict 235 236def do_query(use_source, startdir, srcdir, testdir, petsc_dir, petsc_arch, 237 fields, labels, searchin): 238 """ 239 Do the actual query 240 This part of the code is placed here instead of main() 241 to show how one could translate this into ipython/jupyer notebook 242 commands for more advanced queries 243 """ 244 # Get dictionary 245 if use_source: 246 dataDict=walktree(startdir) 247 else: 248 dataDict=get_gmakegentest_data(testdir, petsc_dir, petsc_arch) 249 250 # Get inverse dictionary for searching 251 invDict=get_inverse_dictionary(dataDict, fields, srcdir) 252 253 # Now do query 254 resList=query(invDict, fields, labels) 255 256 # Filter results using searchin 257 newresList=[] 258 if searchin.strip(): 259 for key in resList: 260 if fnmatch.filter([key],searchin): 261 newresList.append(key) 262 resList=newresList 263 264 # Print in flat list suitable for use by gmakefile.test 265 print(' '.join(resList)) 266 267 return 268 269def main(): 270 parser = optparse.OptionParser(usage="%prog [options] field match_pattern") 271 parser.add_option('-s', '--startdir', dest='startdir', 272 help='Where to start the recursion if not srcdir', 273 default='') 274 parser.add_option('-p', '--petsc-dir', dest='petsc_dir', 275 help='Set PETSC_DIR different from environment', 276 default=os.environ.get('PETSC_DIR')) 277 parser.add_option('-a', '--petsc-arch', dest='petsc_arch', 278 help='Set PETSC_ARCH different from environment', 279 default=os.environ.get('PETSC_ARCH')) 280 parser.add_option('--srcdir', dest='srcdir', 281 help='Set location of sources different from PETSC_DIR/src. Must be full path.', 282 default='src') 283 parser.add_option('-t', '--testdir', dest='testdir', 284 help='Test directory if not PETSC_ARCH/tests. Must be full path', 285 default='tests') 286 parser.add_option('-u', '--use-source', action="store_false", 287 dest='use_source', 288 help='Query all sources rather than those configured in PETSC_ARCH') 289 parser.add_option('-i', '--searchin', dest='searchin', 290 help='Filter results from the arguments', 291 default='') 292 293 opts, args = parser.parse_args() 294 295 # Argument Sanity checks 296 if len(args) != 2: 297 parser.print_usage() 298 print('Arguments: ') 299 print(' field: Field to search for; e.g., requires') 300 print(' To just match names, use "name"') 301 print(' match_pattern: Matching pattern for field; e.g., cuda') 302 return 303 304 # Process arguments and options -- mostly just paths here 305 field=args[0] 306 match=args[1] 307 searchin=opts.searchin 308 309 petsc_dir = opts.petsc_dir 310 petsc_arch = opts.petsc_arch 311 petsc_full_arch = os.path.join(petsc_dir, petsc_arch) 312 313 if opts.srcdir == 'src': 314 petsc_full_src = os.path.join(petsc_dir, 'src') 315 else: 316 petsc_full_src = opts.srcdir 317 if opts.testdir == 'tests': 318 petsc_full_test = os.path.join(petsc_full_arch, 'tests') 319 else: 320 petsc_full_test = opts.testdir 321 if opts.startdir: 322 startdir=opts.startdir=petsc_full_src 323 else: 324 startdir=petsc_full_src 325 326 # Options Sanity checks 327 if not os.path.isdir(petsc_dir): 328 print("PETSC_DIR must be a directory") 329 return 330 331 if not opts.use_source: 332 if not os.path.isdir(petsc_full_arch): 333 print("PETSC_DIR/PETSC_ARCH must be a directory") 334 return 335 elif not os.path.isdir(petsc_full_test): 336 print("Testdir must be a directory"+petsc_full_test) 337 return 338 else: 339 if not os.path.isdir(petsc_full_src): 340 print("Source directory must be a directory"+petsc_full_src) 341 return 342 343 # Do the actual query 344 do_query(opts.use_source, startdir, petsc_full_src, petsc_full_test, 345 petsc_dir, petsc_arch, field, match, searchin) 346 347 return 348 349 350if __name__ == "__main__": 351 main() 352