1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8 9def parse_testargs(file): 10 if os.path.splitext(file)[1] in ['.c', '.cpp']: 11 return sum([[[line.split()[1:], [line.split()[0].strip('//TESTARGS(name=').strip(')')]]] 12 for line in open(file).readlines() 13 if line.startswith('//TESTARGS')], []) 14 elif os.path.splitext(file)[1] == '.usr': 15 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 16 for line in open(file).readlines() 17 if line.startswith('C_TESTARGS')], []) 18 elif os.path.splitext(file)[1] in ['.f90']: 19 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 20 for line in open(file).readlines() 21 if line.startswith('! TESTARGS')], []) 22 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 23 24 25def get_source(test): 26 if test.startswith('petsc-'): 27 return os.path.join('examples', 'petsc', test[6:] + '.c') 28 elif test.startswith('mfem-'): 29 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 30 elif test.startswith('nek-'): 31 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 32 elif test.startswith('fluids-'): 33 return os.path.join('examples', 'fluids', test[7:] + '.c') 34 elif test.startswith('solids-'): 35 return os.path.join('examples', 'solids', test[7:] + '.c') 36 elif test.startswith('ex'): 37 return os.path.join('examples', 'ceed', test + '.c') 38 elif test.endswith('-f'): 39 return os.path.join('tests', test + '.f90') 40 else: 41 return os.path.join('tests', test + '.c') 42 43 44def get_testargs(source): 45 args = parse_testargs(source) 46 if not args: 47 return [(['{ceed_resource}'], [''])] 48 return args 49 50 51def check_required_failure(test_case, stderr, required): 52 if required in stderr: 53 test_case.status = 'fails with required: {}'.format(required) 54 else: 55 test_case.add_failure_info('required: {}'.format(required)) 56 57 58def contains_any(resource, substrings): 59 return any((sub in resource for sub in substrings)) 60 61 62def skip_rule(test, resource): 63 return any(( 64 test.startswith('t4') and contains_any(resource, ['occa']), 65 test.startswith('t5') and contains_any(resource, ['occa']), 66 test.startswith('ex') and contains_any(resource, ['occa']), 67 test.startswith('mfem') and contains_any(resource, ['occa']), 68 test.startswith('nek') and contains_any(resource, ['occa']), 69 test.startswith('petsc-') and contains_any(resource, ['occa']), 70 test.startswith('fluids-') and contains_any(resource, ['occa']), 71 test.startswith('solids-') and contains_any(resource, ['occa']), 72 test.startswith('t318') and contains_any(resource, ['/gpu/cuda/ref']), 73 test.startswith('t506') and contains_any(resource, ['/gpu/cuda/shared']), 74 )) 75 76 77def run(test, backends, mode): 78 import subprocess 79 import time 80 import difflib 81 source = get_source(test) 82 all_args = get_testargs(source) 83 84 if mode.lower() == "tap": 85 print('1..' + str(len(all_args) * len(backends))) 86 87 test_cases = [] 88 my_env = os.environ.copy() 89 my_env["CEED_ERROR_HANDLER"] = 'exit' 90 index = 1 91 for args, name in all_args: 92 for ceed_resource in backends: 93 rargs = [os.path.join('build', test)] + args.copy() 94 rargs[rargs.index('{ceed_resource}')] = ceed_resource 95 96 # run test 97 if skip_rule(test, ceed_resource): 98 test_case = TestCase('{} {}'.format(test, ceed_resource), 99 elapsed_sec=0, 100 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 101 stdout='', 102 stderr='') 103 test_case.add_skipped_info('Pre-run skip rule') 104 else: 105 start = time.time() 106 proc = subprocess.run(rargs, 107 stdout=subprocess.PIPE, 108 stderr=subprocess.PIPE, 109 env=my_env) 110 proc.stdout = proc.stdout.decode('utf-8') 111 proc.stderr = proc.stderr.decode('utf-8') 112 113 test_case = TestCase('{} {} {}'.format(test, *name, ceed_resource), 114 classname=os.path.dirname(source), 115 elapsed_sec=time.time() - start, 116 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 117 stdout=proc.stdout, 118 stderr=proc.stderr) 119 ref_stdout = os.path.join('tests/output', test + '.out') 120 121 # check for allowed errors 122 if not test_case.is_skipped() and proc.stderr: 123 if 'OCCA backend failed to use' in proc.stderr: 124 test_case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 125 elif 'Backend does not implement' in proc.stderr: 126 test_case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 127 elif 'Can only provide HOST memory for this backend' in proc.stderr: 128 test_case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 129 elif 'Test not implemented in single precision' in proc.stderr: 130 test_case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 131 132 # check required failures 133 if not test_case.is_skipped(): 134 if test[:4] in 't006 t007'.split(): 135 check_required_failure(test_case, proc.stderr, 'No suitable backend:') 136 if test[:4] in 't008'.split(): 137 check_required_failure(test_case, proc.stderr, 'Available backend resources:') 138 if test[:4] in 't110 t111 t112 t113 t114'.split(): 139 check_required_failure(test_case, proc.stderr, 'Cannot grant CeedVector array access') 140 if test[:4] in 't115'.split(): 141 check_required_failure(test_case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 142 if test[:4] in 't116'.split(): 143 check_required_failure(test_case, proc.stderr, 'Cannot destroy CeedVector, the writable access lock is in use') 144 if test[:4] in 't117'.split(): 145 check_required_failure(test_case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 146 if test[:4] in 't118'.split(): 147 check_required_failure(test_case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 148 if test[:4] in 't215'.split(): 149 check_required_failure(test_case, proc.stderr, 'Cannot destroy CeedElemRestriction, a process has read access to the offset data') 150 if test[:4] in 't303'.split(): 151 check_required_failure(test_case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 152 if test[:4] in 't408'.split(): 153 check_required_failure(test_case, proc.stderr, 'CeedQFunctionContextGetData(): Cannot grant CeedQFunctionContext data access, a process has read access') 154 if test[:4] in 't409'.split() and contains_any(ceed_resource, ['memcheck']): 155 check_required_failure(test_case, proc.stderr, 'Context data changed while accessed in read-only mode') 156 157 # classify other results 158 if not test_case.is_skipped() and not test_case.status: 159 if proc.stderr: 160 test_case.add_failure_info('stderr', proc.stderr) 161 elif proc.returncode != 0: 162 test_case.add_error_info('returncode = {}'.format(proc.returncode)) 163 elif os.path.isfile(ref_stdout): 164 with open(ref_stdout) as ref: 165 diff = list(difflib.unified_diff(ref.readlines(), 166 proc.stdout.splitlines(keepends=True), 167 fromfile=ref_stdout, 168 tofile='New')) 169 if diff: 170 test_case.add_failure_info('stdout', output=''.join(diff)) 171 elif proc.stdout and test[:4] not in 't003': 172 test_case.add_failure_info('stdout', output=proc.stdout) 173 174 # store result 175 test_case.args = ' '.join(rargs) 176 test_cases.append(test_case) 177 178 if mode.lower() == "tap": 179 # print incremental output if TAP mode 180 print('# Test: {}'.format(test_case.name.split(' ')[1])) 181 print('# $ {}'.format(test_case.args)) 182 if test_case.is_error(): 183 print('not ok {} - ERROR: {}'.format(index, test_case.errors[0]['message'].strip())) 184 print(test_case.errors[0]['output'].strip()) 185 elif test_case.is_failure(): 186 print('not ok {} - FAIL: {}'.format(index, test_case.failures[0]['message'].strip())) 187 print(test_case.failures[0]['output'].strip()) 188 elif test_case.is_skipped(): 189 print('ok {} - SKIP: {}'.format(index, test_case.skipped[0]['message'].strip())) 190 else: 191 print('ok {} - PASS'.format(index)) 192 sys.stdout.flush() 193 else: 194 # print error or failure information if JUNIT mode 195 if test_case.is_error(): 196 print('Test: {} {}'.format(test_case.name.split(' ')[0], test_case.name.split(' ')[1])) 197 print('ERROR: {}'.format(test_case.errors[0]['message'].strip())) 198 print(test_case.errors[0]['output'].strip()) 199 elif test_case.is_failure(): 200 print('Test: {} {}'.format(test_case.name.split(' ')[0], test_case.name.split(' ')[1])) 201 print('FAIL: {}'.format(test_case.failures[0]['message'].strip())) 202 print(test_case.failures[0]['output'].strip()) 203 sys.stdout.flush() 204 index += 1 205 206 return TestSuite(test, test_cases) 207 208if __name__ == '__main__': 209 import argparse 210 parser = argparse.ArgumentParser('Test runner with JUnit and TAP output') 211 parser.add_argument('--mode', help='Output mode, JUnit or TAP', default="JUnit") 212 parser.add_argument('--output', help='Output file to write test', default=None) 213 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 214 parser.add_argument('test', help='Test executable', nargs='?') 215 args = parser.parse_args() 216 217 if args.gather: 218 gather() 219 else: 220 backends = os.environ['BACKENDS'].split() 221 222 # run tests 223 result = run(args.test, backends, args.mode) 224 225 # build output 226 if args.mode.lower() == "junit": 227 junit_batch = '' 228 try: 229 junit_batch = '-' + os.environ['JUNIT_BATCH'] 230 except: 231 pass 232 output = (os.path.join('build', args.test + junit_batch + '.junit') 233 if args.output is None 234 else args.output) 235 236 with open(output, 'w') as fd: 237 TestSuite.to_file(fd, [result]) 238 elif args.mode.lower() != "tap": 239 raise Exception("output mode not recognized") 240 241 # check return code 242 for t in result.test_cases: 243 failures = len([c for c in result.test_cases if c.is_failure()]) 244 errors = len([c for c in result.test_cases if c.is_error()]) 245 if failures + errors > 0 and args.mode.lower() != "tap": 246 sys.exit(1) 247