1#!/usr/bin/env python 2import fnmatch 3import glob 4import inspect 5import os 6import optparse 7import pickle 8import re 9import sys 10 11thisfile = os.path.abspath(inspect.getfile(inspect.currentframe())) 12pdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(thisfile))))) 13sys.path.insert(0, os.path.join(pdir, 'config')) 14 15import testparse 16from gmakegentest import nameSpace 17 18 19""" 20 Tool for querying the tests. 21 22 Which tests to query? Two options: 23 1. Query only the tests that are run for a given configuration. 24 2. Query all of the test files in the source directory 25 For #1: 26 Use dataDict as written out by gmakegentest.py in $PETSC_ARCH/$TESTBASE 27 For #2: 28 Walk the entire tree parsing the files as we go along using testparse. 29 The tree walker is simpler than what is in gmakegentest.py 30 31 The dataDict follows that generated by testparse. gmakegentest.py does 32 further manipulations of the dataDict to handle things like for loops 33 so if using #2, those modifications are not included. 34 35 Querying: 36 The dataDict dictionary is then "inverted" to create a dictionary with the 37 range of field values as keys and list test names as the values. This 38 allows fast searching 39 40""" 41 42def isFile(maybeFile): 43 ext=os.path.splitext(maybeFile)[1] 44 if not ext: return False 45 if ext not in ['.c','.cxx','.cpp','F90','F','cu']: return False 46 return True 47 48def pathToLabel(path): 49 """ 50 Because the scripts have a non-unique naming, the pretty-printing 51 needs to convey the srcdir and srcfile. There are two ways of doing this. 52 """ 53 # Strip off any top-level directories or spaces 54 path=path.strip().replace(pdir,'') 55 path=path.replace('src/','') 56 if isFile(path): 57 prefix=os.path.dirname(path).replace("/","_") 58 suffix=os.path.splitext(os.path.basename(path))[0] 59 label=prefix+"-"+suffix+'_*' 60 else: 61 path=path.rstrip('/') 62 label=path.replace("/","_").replace('tests_','tests-').replace('tutorials_','tutorials-') 63 return label 64 65def get_value(varset): 66 """ 67 Searching args is a bit funky: 68 Consider 69 args: -ksp_monitor_short -pc_type ml -ksp_max_it 3 70 Search terms are: 71 ksp_monitor, 'pc_type ml', ksp_max_it 72 Also ignore all loops 73 -pc_fieldsplit_diag_use_amat {{0 1}} 74 Gives: pc_fieldsplit_diag_use_amat as the search term 75 Also ignore -f ... (use matrices from file) because I'll assume 76 that this kind of information isn't needed for testing. If it's 77 a separate search than just grep it 78 """ 79 if varset.startswith('-f '): return None 80 81 # First remove loops 82 value=re.sub('{{.*}}','',varset) 83 # Next remove - 84 value=varset.lstrip("-") 85 # Get rid of numbers 86 value=re.sub(r"[+-]? *(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?",'',value) 87 # return without spaces 88 return value.strip() 89 90def query(invDict,fields,labels): 91 """ 92 Search the keys using fnmatch to find matching names and return list with 93 the results 94 """ 95 setlist=[] # setlist is a list of lists that set opertions will operate on 96 llist=labels.replace('|',',').split(',') 97 i=-1 98 for field in fields.replace('|',',').split(','): 99 i+=1 100 label=llist[i] 101 if field == 'name': 102 if '/' in label: 103 label=pathToLabel(label) 104 elif label.startswith('src'): 105 label=label.lstrip('src').lstrip('*') 106 setlist.append(fnmatch.filter(invDict['name'],label)) 107 continue 108 109 foundLabel=False # easy to do if you misspell argument search 110 for key in invDict[field]: 111 if fnmatch.filter([key],label): 112 foundLabel=True 113 # Do not return values with not unless label itself has not 114 if label.startswith('!') and not key.startswith('!'): continue 115 if not label.startswith('!') and key.startswith('!'): continue 116 setlist.append(invDict[field][key]) 117 if not foundLabel: 118 setlist.append([]) 119 120 # Now process the union and intersection operators based on setlist 121 allresults=[] 122 # Union 123 i=-1 124 for ufield in fields.split(','): 125 i+=1 126 if '|' in ufield: 127 # Intersection 128 label=llist[i] 129 results=set(setlist[i]) 130 for field in ufield.split('|')[1:]: 131 i+=1 132 label=llist[i] 133 results=results.intersection(set(setlist[i])) 134 allresults+=list(results) 135 else: 136 allresults+=setlist[i] 137 138 # remove duplicate entries and sort to give consistent results 139 uniqlist=list(set(allresults)) 140 uniqlist.sort() 141 return uniqlist 142 143def get_inverse_dictionary(dataDict,fields,srcdir): 144 """ 145 Create a dictionary with the values of field as the keys, and the name of 146 the tests as the results. 147 """ 148 invDict={} 149 # Comma-delimited lists denote union 150 for field in fields.replace('|',',').split(','): 151 if field not in invDict: 152 if field == 'name': 153 invDict[field]=[] # List for ease 154 else: 155 invDict[field]={} 156 for root in dataDict: 157 for exfile in dataDict[root]: 158 for test in dataDict[root][exfile]: 159 if test in testparse.buildkeys: continue 160 defroot = testparse.getDefaultOutputFileRoot(test) 161 fname=nameSpace(defroot,os.path.relpath(root,srcdir)) 162 if field == 'name': 163 invDict['name'].append(fname) 164 continue 165 if field not in dataDict[root][exfile][test]: continue 166 values=dataDict[root][exfile][test][field] 167 168 if not field == 'args' and not field == 'diff_args': 169 for val in values.split(): 170 if val in invDict[field]: 171 invDict[field][val].append(fname) 172 else: 173 invDict[field][val] = [fname] 174 else: 175 # Args are funky. 176 for varset in re.split('(^|\W)-(?=[a-zA-Z])',values): 177 val=get_value(varset) 178 if not val: continue 179 if val in invDict[field]: 180 invDict[field][val].append(fname) 181 else: 182 invDict[field][val] = [fname] 183 # remove duplicate entries (multiple test/file) 184 if not field == 'name': 185 for val in invDict[field]: 186 invDict[field][val]=list(set(invDict[field][val])) 187 188 return invDict 189 190def get_gmakegentest_data(testdir,petsc_dir,petsc_arch): 191 """ 192 Write out the dataDict into a pickle file 193 """ 194 # This needs to be consistent with gmakegentest.py of course 195 pkl_file=os.path.join(testdir,'datatest.pkl') 196 # If it doesn't exist, then we need to regenerate 197 if not os.path.exists(pkl_file): 198 startdir=os.path.abspath(os.curdir) 199 os.chdir(petsc_dir) 200 args='--petsc-dir='+petsc_dir+' --petsc-arch='+petsc_arch+' --testdir='+testdir 201 buf = os.popen('config/gmakegentest.py '+args).read() 202 os.chdir(startdir) 203 204 fd = open(pkl_file, 'rb') 205 dataDict=pickle.load(fd) 206 fd.close() 207 return dataDict 208 209def walktree(top): 210 """ 211 Walk a directory tree, starting from 'top' 212 """ 213 verbose = False 214 dataDict = {} 215 alldatafiles = [] 216 for root, dirs, files in os.walk(top, topdown=False): 217 if root == 'output': continue 218 if '.dSYM' in root: continue 219 if verbose: print(root) 220 221 dataDict[root] = {} 222 223 for exfile in files: 224 # Ignore emacs files 225 if exfile.startswith("#") or exfile.startswith(".#"): continue 226 ext=os.path.splitext(exfile)[1] 227 if ext[1:] not in ['c','cxx','cpp','cu','F90','F']: continue 228 229 # Convenience 230 fullex = os.path.join(root, exfile) 231 if verbose: print(' --> '+fullex) 232 dataDict[root].update(testparse.parseTestFile(fullex, 0)) 233 234 return dataDict 235 236def do_query(use_source, startdir, srcdir, testdir, petsc_dir, petsc_arch, 237 fields, labels, searchin): 238 """ 239 Do the actual query 240 This part of the code is placed here instead of main() 241 to show how one could translate this into ipython/jupyer notebook 242 commands for more advanced queries 243 """ 244 # Get dictionary 245 if use_source: 246 dataDict=walktree(startdir) 247 else: 248 dataDict=get_gmakegentest_data(testdir, petsc_dir, petsc_arch) 249 250 # Get inverse dictionary for searching 251 invDict=get_inverse_dictionary(dataDict, fields, srcdir) 252 253 # Now do query 254 resList=query(invDict, fields, labels) 255 256 # Filter results using searchin 257 newresList=[] 258 if searchin.strip(): 259 if not searchin.startswith('!'): 260 for key in resList: 261 if fnmatch.filter([key],searchin): 262 newresList.append(key) 263 else: 264 for key in resList: 265 if not fnmatch.filter([key],searchin[1:]): 266 newresList.append(key) 267 resList=newresList 268 269 # Print in flat list suitable for use by gmakefile.test 270 print(' '.join(resList)) 271 272 return 273 274def expand_path_like(petscdir,petscarch,pathlike): 275 def remove_prefix(text,prefix): 276 return text[text.startswith(prefix) and len(prefix):] 277 278 # expand user second, as expandvars may insert a '~' 279 string = os.path.expanduser(os.path.expandvars(pathlike)) 280 # if the dirname check succeeds then likely we have a glob expression 281 pardir = os.path.dirname(string) 282 if os.path.exists(pardir): 283 suffix = string.replace(pardir,'') # get whatever is left over 284 pathlike = remove_prefix(os.path.relpath(os.path.abspath(pardir),petscdir),'.'+os.path.sep) 285 if petscarch == '': 286 pathlike = pathlike.replace(os.path.sep.join(('share','petsc','examples'))+'/','') 287 pathlike += suffix 288 return pathlike 289 290def main(): 291 parser = optparse.OptionParser(usage="%prog [options] field match_pattern") 292 parser.add_option('-s', '--startdir', dest='startdir', 293 help='Where to start the recursion if not srcdir', 294 default='') 295 parser.add_option('-p', '--petsc-dir', dest='petsc_dir', 296 help='Set PETSC_DIR different from environment', 297 default=os.environ.get('PETSC_DIR')) 298 parser.add_option('-a', '--petsc-arch', dest='petsc_arch', 299 help='Set PETSC_ARCH different from environment', 300 default=os.environ.get('PETSC_ARCH')) 301 parser.add_option('--srcdir', dest='srcdir', 302 help='Set location of sources different from PETSC_DIR/src. Must be full path.', 303 default='src') 304 parser.add_option('-t', '--testdir', dest='testdir', 305 help='Test directory if not PETSC_ARCH/tests. Must be full path', 306 default='tests') 307 parser.add_option('-u', '--use-source', action="store_false", 308 dest='use_source', 309 help='Query all sources rather than those configured in PETSC_ARCH') 310 parser.add_option('-i', '--searchin', dest='searchin', 311 help='Filter results from the arguments', 312 default='') 313 314 opts, args = parser.parse_args() 315 316 # Argument Sanity checks 317 if len(args) != 2: 318 parser.print_usage() 319 print('Arguments: ') 320 print(' field: Field to search for; e.g., requires') 321 print(' To just match names, use "name"') 322 print(' match_pattern: Matching pattern for field; e.g., cuda') 323 return 324 325 # Process arguments and options -- mostly just paths here 326 field=args[0] 327 match=args[1] 328 searchin=opts.searchin 329 330 petsc_dir = opts.petsc_dir 331 petsc_arch = opts.petsc_arch 332 petsc_full_arch = os.path.join(petsc_dir, petsc_arch) 333 334 if petsc_arch == '': 335 petsc_full_src = os.path.join(petsc_dir, 'share', 'petsc', 'examples', 'src') 336 else: 337 if opts.srcdir == 'src': 338 petsc_full_src = os.path.join(petsc_dir, 'src') 339 else: 340 petsc_full_src = opts.srcdir 341 if opts.testdir == 'tests': 342 petsc_full_test = os.path.join(petsc_full_arch, 'tests') 343 else: 344 petsc_full_test = opts.testdir 345 if opts.startdir: 346 startdir=opts.startdir=petsc_full_src 347 else: 348 startdir=petsc_full_src 349 350 # Options Sanity checks 351 if not os.path.isdir(petsc_dir): 352 print("PETSC_DIR must be a directory") 353 return 354 355 if not opts.use_source: 356 if not os.path.isdir(petsc_full_arch): 357 print("PETSC_DIR/PETSC_ARCH must be a directory") 358 return 359 elif not os.path.isdir(petsc_full_test): 360 print("Testdir must be a directory"+petsc_full_test) 361 return 362 else: 363 if not os.path.isdir(petsc_full_src): 364 print("Source directory must be a directory"+petsc_full_src) 365 return 366 367 match = expand_path_like(petsc_dir,petsc_arch,match) 368 369 # Do the actual query 370 do_query(opts.use_source, startdir, petsc_full_src, petsc_full_test, 371 petsc_dir, petsc_arch, field, match, searchin) 372 373 return 374 375 376if __name__ == "__main__": 377 main() 378