1#!/usr/bin/env python 2import fnmatch 3import glob 4import inspect 5import os 6import optparse 7import pickle 8import re 9import sys 10 11thisfile = os.path.abspath(inspect.getfile(inspect.currentframe())) 12pdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(thisfile))))) 13sys.path.insert(0, os.path.join(pdir, 'config')) 14 15import testparse 16from gmakegentest import nameSpace 17 18 19""" 20 Tool for querying the tests. 21 22 Which tests to query? Two options: 23 1. Query only the tests that are run for a given configuration. 24 2. Query all of the test files in the source directory 25 For #1: 26 Use dataDict as written out by gmakegentest.py in $PETSC_ARCH/$TESTBASE 27 For #2: 28 Walk the entire tree parsing the files as we go along using testparse. 29 The tree walker is simpler than what is in gmakegentest.py 30 31 The dataDict follows that generated by testparse. gmakegentest.py does 32 further manipulations of the dataDict to handle things like for loops 33 so if using #2, those modifications are not included. 34 35 Querying: 36 The dataDict dictionary is then "inverted" to create a dictionary with the 37 range of field values as keys and list test names as the values. This 38 allows fast searching 39 40""" 41 42def query(invDict,label): 43 """ 44 Search the keys using fnmatch to find matching names and return list with 45 the results 46 """ 47 results=[] 48 if 'name' in invDict: 49 return fnmatch.filter(invDict['name'],label) 50 51 for key in invDict: 52 if fnmatch.filter([key],label): 53 # Do not return values with not unless label itself has not 54 if label.startswith('!') and not key.startswith('!'): continue 55 if not label.startswith('!') and key.startswith('!'): continue 56 results += invDict[key] 57 58 return results 59 60def get_inverse_dictionary(dataDict,field,srcdir): 61 """ 62 Create a dictionary with the values of field as the keys, and the name of 63 the tests as the results. 64 """ 65 invDict={} 66 if field == 'name': invDict['name']=[] 67 for root in dataDict: 68 for exfile in dataDict[root]: 69 for test in dataDict[root][exfile]: 70 if test in testparse.buildkeys: continue 71 defroot = testparse.getDefaultOutputFileRoot(test) 72 name=nameSpace(defroot,os.path.relpath(root,srcdir)) 73 if field == 'name': 74 invDict['name'].append(name) 75 continue 76 if field not in dataDict[root][exfile][test]: continue 77 values=dataDict[root][exfile][test][field] 78 79 for val in values.split(): 80 if val in invDict: 81 invDict[val].append(name) 82 else: 83 invDict[val] = [name] 84 return invDict 85 86def get_gmakegentest_data(testdir,petsc_dir,petsc_arch): 87 """ 88 Write out the dataDict into a pickle file 89 """ 90 # This needs to be consistent with gmakegentest.py of course 91 pkl_file=os.path.join(testdir,'datatest.pkl') 92 # If it doesn't exist, then we need to regenerate 93 if not os.path.exists(pkl_file): 94 startdir=os.path.abspath(os.curdir) 95 os.chdir(petsc_dir) 96 args='--petsc-dir='+petsc_dir+' --petsc-arch='+petsc_arch+' --testdir='+testdir 97 buf = os.popen('config/gmakegentest.py '+args).read() 98 os.chdir(startdir) 99 100 fd = open(pkl_file, 'rb') 101 dataDict=pickle.load(fd) 102 fd.close() 103 return dataDict 104 105def walktree(top): 106 """ 107 Walk a directory tree, starting from 'top' 108 """ 109 verbose = False 110 dataDict = {} 111 alldatafiles = [] 112 for root, dirs, files in os.walk(top, topdown=False): 113 if "examples" not in root: continue 114 if root == 'output': continue 115 if '.dSYM' in root: continue 116 if verbose: print(root) 117 118 dataDict[root] = {} 119 120 for exfile in files: 121 # Ignore emacs files 122 if exfile.startswith("#") or exfile.startswith(".#"): continue 123 ext=os.path.splitext(exfile)[1] 124 if ext[1:] not in ['c','cxx','cpp','cu','F90','F']: continue 125 126 # Convenience 127 fullex = os.path.join(root, exfile) 128 if verbose: print(' --> '+fullex) 129 dataDict[root].update(testparse.parseTestFile(fullex, 0)) 130 131 return dataDict 132 133def do_query(use_source, startdir, srcdir, testdir, petsc_dir, petsc_arch, field, label): 134 """ 135 Do the actual query 136 This part of the code is placed here instead of main() 137 to show how one could translate this into ipython/jupyer notebook 138 commands for more advanced queries 139 """ 140 # Get dictionary 141 if use_source: 142 dataDict=walktree(startdir) 143 else: 144 dataDict=get_gmakegentest_data(testdir, petsc_dir, petsc_arch) 145 146 # Get inverse dictionary for searching 147 invDict=get_inverse_dictionary(dataDict, field, srcdir) 148 #print(invDict) 149 150 # Now do query 151 resList=query(invDict, label) 152 153 # Print in flat list suitable for use by gmakefile.test 154 print(' '.join(resList)) 155 156 return 157 158def main(): 159 parser = optparse.OptionParser(usage="%prog [options] field match_pattern") 160 parser.add_option('-s', '--startdir', dest='startdir', 161 help='Where to start the recursion if not srcdir', 162 default='') 163 parser.add_option('-p', '--petsc-dir', dest='petsc_dir', 164 help='Set PETSC_DIR different from environment', 165 default=os.environ.get('PETSC_DIR')) 166 parser.add_option('-a', '--petsc-arch', dest='petsc_arch', 167 help='Set PETSC_ARCH different from environment', 168 default=os.environ.get('PETSC_ARCH')) 169 parser.add_option('--srcdir', dest='srcdir', 170 help='Set location of sources different from PETSC_DIR/src. Must be full path.', 171 default='src') 172 parser.add_option('-t', '--testdir', dest='testdir', 173 help='Test directory if not PETSC_ARCH/tests. Must be full path', 174 default='tests') 175 parser.add_option('-u', '--use-source', action="store_false", 176 dest='use_source', 177 help='Query all sources rather than those configured in PETSC_ARCH') 178 179 opts, args = parser.parse_args() 180 181 # Argument Sanity checks 182 if len(args) != 2: 183 parser.print_usage() 184 print('Arguments: ') 185 print(' field: Field to search for; e.g., requires') 186 print(' To just match names, use "name"') 187 print(' match_pattern: Matching pattern for field; e.g., cuda') 188 return 189 190 # Process arguments and options -- mostly just paths here 191 field=args[0] 192 match=args[1] 193 194 petsc_dir = opts.petsc_dir 195 petsc_arch = opts.petsc_arch 196 petsc_full_arch = os.path.join(petsc_dir, petsc_arch) 197 198 if opts.srcdir == 'src': 199 petsc_full_src = os.path.join(petsc_dir, 'src') 200 else: 201 petsc_full_src = opts.srcdir 202 if opts.testdir == 'tests': 203 petsc_full_test = os.path.join(petsc_full_arch, 'tests') 204 else: 205 petsc_full_test = opts.testdir 206 if opts.startdir: 207 startdir=opts.startdir=petsc_full_src 208 else: 209 startdir=petsc_full_src 210 211 # Options Sanity checks 212 if not os.path.isdir(petsc_dir): 213 print("PETSC_DIR must be a directory") 214 return 215 216 if not opts.use_source: 217 if not os.path.isdir(petsc_full_arch): 218 print("PETSC_DIR/PETSC_ARCH must be a directory") 219 return 220 elif not os.path.isdir(petsc_full_test): 221 print("Testdir must be a directory"+petsc_full_test) 222 return 223 else: 224 if not os.path.isdir(petsc_full_src): 225 print("Source directory must be a directory"+petsc_full_src) 226 return 227 228 # Do the actual query 229 do_query(opts.use_source, startdir, petsc_full_src, petsc_full_test, 230 petsc_dir, petsc_arch, field, match) 231 232 return 233 234 235if __name__ == "__main__": 236 main() 237