1#!/usr/bin/env python 2import fnmatch 3import glob 4import inspect 5import os 6import optparse 7import pickle 8import re 9import sys 10 11thisfile = os.path.abspath(inspect.getfile(inspect.currentframe())) 12pdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(thisfile))))) 13sys.path.insert(0, os.path.join(pdir, 'config')) 14 15import testparse 16from gmakegentest import nameSpace 17 18 19""" 20 Tool for querying the tests. 21 22 Which tests to query? Two options: 23 1. Query only the tests that are run for a given configuration. 24 2. Query all of the test files in the source directory 25 For #1: 26 Use dataDict as written out by gmakegentest.py in $PETSC_ARCH/$TESTBASE 27 For #2: 28 Walk the entire tree parsing the files as we go along using testparse. 29 The tree walker is simpler than what is in gmakegentest.py 30 31 The dataDict follows that generated by testparse. gmakegentest.py does 32 further manipulations of the dataDict to handle things like for loops 33 so if using #2, those modifications are not included. 34 35 Querying: 36 The dataDict dictionary is then "inverted" to create a dictionary with the 37 range of field values as keys and list test names as the values. This 38 allows fast searching 39 40""" 41 42def query(invDict,label): 43 """ 44 Search the keys using fnmatch to find matching names and return list with 45 the results 46 """ 47 results=[] 48 for key in invDict: 49 if fnmatch.filter([key],label): 50 # Do not return values with not unless label itself has not 51 if label.startswith('!') and not key.startswith('!'): continue 52 if not label.startswith('!') and key.startswith('!'): continue 53 results += invDict[key] 54 55 return results 56 57def get_inverse_dictionary(dataDict,field,srcdir): 58 """ 59 Create a dictionary with the values of field as the keys, and the name of 60 the tests as the results. 61 """ 62 invDict={} 63 for root in dataDict: 64 for exfile in dataDict[root]: 65 for test in dataDict[root][exfile]: 66 if field not in dataDict[root][exfile][test]: continue 67 defroot = testparse.getDefaultOutputFileRoot(test) 68 name=nameSpace(defroot,os.path.relpath(root,srcdir)) 69 values=dataDict[root][exfile][test][field] 70 71 for val in values.split(): 72 if val in invDict: 73 invDict[val].append(name) 74 else: 75 invDict[val] = [name] 76 return invDict 77 78def get_gmakegentest_data(testdir): 79 """ 80 Write out the dataDict into a pickle file 81 """ 82 # This needs to be consistent with gmakegentest.py of course 83 fd = open(os.path.join(testdir,'datatest.pkl'), 'rb') 84 dataDict=pickle.load(fd) 85 fd.close() 86 return dataDict 87 88def walktree(top): 89 """ 90 Walk a directory tree, starting from 'top' 91 """ 92 verbose = False 93 dataDict = {} 94 alldatafiles = [] 95 for root, dirs, files in os.walk(top, topdown=False): 96 if "examples" not in root: continue 97 if root == 'output': continue 98 if '.dSYM' in root: continue 99 if verbose: print(root) 100 101 dataDict[root] = {} 102 103 for exfile in files: 104 # Ignore emacs files 105 if exfile.startswith("#") or exfile.startswith(".#"): continue 106 ext=os.path.splitext(exfile)[1] 107 if ext[1:] not in ['c','cxx','cpp','cu','F90','F']: continue 108 109 # Convenience 110 fullex = os.path.join(root, exfile) 111 if verbose: print(' --> '+fullex) 112 dataDict[root].update(testparse.parseTestFile(fullex, 0)) 113 114 return dataDict 115 116def do_query(use_source, startdir, srcdir, testdir, field, label): 117 """ 118 Do the actual query 119 This part of the code is placed here instead of main() 120 to show how one could translate this into ipython/jupyer notebook 121 commands for more advanced queries 122 """ 123 # Get dictionary 124 if use_source: 125 dataDict=walktree(startdir) 126 else: 127 dataDict=get_gmakegentest_data(testdir) 128 129 # Get inverse dictionary for searching 130 invDict=get_inverse_dictionary(dataDict, field, srcdir) 131 132 # Now do query 133 resList=query(invDict, label) 134 135 # Print in flat list suitable for use by gmakefile.test 136 print(' '.join(resList)) 137 138 return 139 140def main(): 141 parser = optparse.OptionParser(usage="%prog [options] field match_pattern") 142 parser.add_option('-s', '--startdir', dest='startdir', 143 help='Where to start the recursion if not srcdir', 144 default='') 145 parser.add_option('-p', '--petsc_dir', dest='petsc_dir', 146 help='Set PETSC_ARCH different from environment', 147 default=os.environ.get('PETSC_DIR')) 148 parser.add_option('-a', '--petsc-arch', dest='petsc_arch', 149 help='Set PETSC_ARCH different from environment', 150 default=os.environ.get('PETSC_ARCH')) 151 parser.add_option('--srcdir', dest='srcdir', 152 help='Set location of sources different from PETSC_DIR/src. Must be full path.', 153 default='src') 154 parser.add_option('-t', '--testdir', dest='testdir', 155 help='Test directory if not PETSC_ARCH/tests. Must be full path', 156 default='tests') 157 parser.add_option('-u', '--use-source', action="store_false", 158 dest='use_source', 159 help='Query all sources rather than those configured in PETSC_ARCH') 160 161 opts, args = parser.parse_args() 162 163 # Argument Sanity checks 164 if len(args) != 2: 165 parser.print_usage() 166 print('Arguments: ') 167 print(' field: Field to search for; e.g., requires') 168 print(' match_pattern: Matching pattern for field; e.g., cuda') 169 return 170 171 # Process arguments and options -- mostly just paths here 172 field=args[0] 173 match=args[1] 174 175 petsc_dir = opts.petsc_dir 176 petsc_arch = opts.petsc_arch 177 petsc_full_arch = os.path.join(petsc_dir, petsc_arch) 178 179 if opts.srcdir == 'src': 180 petsc_full_src = os.path.join(petsc_dir, 'src') 181 else: 182 petsc_full_src = opts.srcdir 183 if opts.testdir == 'tests': 184 petsc_full_test = os.path.join(petsc_full_arch, 'tests') 185 else: 186 petsc_full_test = opts.testdir 187 if opts.startdir: 188 startdir=opts.startdir=petsc_full_src 189 else: 190 startdir=petsc_full_src 191 192 # Options Sanity checks 193 if not os.path.isdir(petsc_dir): 194 print("PETSC_DIR must be a directory") 195 return 196 197 if not opts.use_source: 198 if not os.path.isdir(petsc_full_arch): 199 print("PETSC_DIR/PETSC_ARCH must be a directory") 200 return 201 elif not os.path.isdir(petsc_full_test): 202 print("Testdir must be a directory"+petsc_full_test) 203 return 204 else: 205 if not os.path.isdir(petsc_full_src): 206 print("Source directory must be a directory"+petsc_full_src) 207 return 208 209 # Do the actual query 210 do_query(opts.use_source, startdir, petsc_full_src, petsc_full_test, field, match) 211 212 return 213 214 215if __name__ == "__main__": 216 main() 217