1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8 9def parse_testargs(file): 10 if os.path.splitext(file)[1] in ['.c', '.cpp']: 11 return sum([[[line.split()[1:], [line.split()[0].strip('//TESTARGS(name=').strip(')')]]] 12 for line in open(file).readlines() 13 if line.startswith('//TESTARGS')], []) 14 elif os.path.splitext(file)[1] == '.usr': 15 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 16 for line in open(file).readlines() 17 if line.startswith('C_TESTARGS')], []) 18 elif os.path.splitext(file)[1] in ['.f90']: 19 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 20 for line in open(file).readlines() 21 if line.startswith('! TESTARGS')], []) 22 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 23 24 25def get_source(test): 26 if test.startswith('petsc-'): 27 return os.path.join('examples', 'petsc', test[6:] + '.c') 28 elif test.startswith('mfem-'): 29 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 30 elif test.startswith('nek-'): 31 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 32 elif test.startswith('fluids-'): 33 return os.path.join('examples', 'fluids', test[7:] + '.c') 34 elif test.startswith('solids-'): 35 return os.path.join('examples', 'solids', test[7:] + '.c') 36 elif test.startswith('ex'): 37 return os.path.join('examples', 'ceed', test + '.c') 38 elif test.endswith('-f'): 39 return os.path.join('tests', test + '.f90') 40 else: 41 return os.path.join('tests', test + '.c') 42 43 44def get_testargs(source): 45 args = parse_testargs(source) 46 if not args: 47 return [(['{ceed_resource}'], [''])] 48 return args 49 50 51def check_required_failure(test_case, stderr, required): 52 if required in stderr: 53 test_case.status = 'fails with required: {}'.format(required) 54 else: 55 test_case.add_failure_info('required: {}'.format(required)) 56 57 58def contains_any(resource, substrings): 59 return any((sub in resource for sub in substrings)) 60 61 62def skip_rule(test, resource): 63 return any(( 64 test.startswith('t4') and contains_any(resource, ['occa']), 65 test.startswith('t5') and contains_any(resource, ['occa']), 66 test.startswith('ex') and contains_any(resource, ['occa']), 67 test.startswith('mfem') and contains_any(resource, ['occa']), 68 test.startswith('nek') and contains_any(resource, ['occa']), 69 test.startswith('petsc-') and contains_any(resource, ['occa']), 70 test.startswith('fluids-') and contains_any(resource, ['occa']), 71 test.startswith('solids-') and contains_any(resource, ['occa']), 72 test.startswith('t318') and contains_any(resource, ['/gpu/cuda/ref']), 73 test.startswith('t506') and contains_any(resource, ['/gpu/cuda/shared']), 74 )) 75 76 77def run(test, backends, mode): 78 import subprocess 79 import time 80 import difflib 81 source = get_source(test) 82 all_args = get_testargs(source) 83 84 if mode.lower() == "tap": 85 print('1..' + str(len(all_args) * len(backends))) 86 87 test_cases = [] 88 my_env = os.environ.copy() 89 my_env["CEED_ERROR_HANDLER"] = 'exit' 90 index = 1 91 for args, name in all_args: 92 for ceed_resource in backends: 93 rargs = [os.path.join('build', test)] + args.copy() 94 rargs[rargs.index('{ceed_resource}')] = ceed_resource 95 96 # run test 97 if skip_rule(test, ceed_resource): 98 test_case = TestCase('{} {}'.format(test, ceed_resource), 99 elapsed_sec=0, 100 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 101 stdout='', 102 stderr='') 103 test_case.add_skipped_info('Pre-run skip rule') 104 else: 105 start = time.time() 106 proc = subprocess.run(rargs, 107 stdout=subprocess.PIPE, 108 stderr=subprocess.PIPE, 109 env=my_env) 110 proc.stdout = proc.stdout.decode('utf-8') 111 proc.stderr = proc.stderr.decode('utf-8') 112 113 test_case = TestCase('{} {} {}'.format(test, *name, ceed_resource), 114 classname=os.path.dirname(source), 115 elapsed_sec=time.time() - start, 116 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 117 stdout=proc.stdout, 118 stderr=proc.stderr) 119 ref_stdout = os.path.join('tests/output', test + '.out') 120 121 # check for allowed errors 122 if not test_case.is_skipped() and proc.stderr: 123 if 'OCCA backend failed to use' in proc.stderr: 124 test_case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 125 elif 'Backend does not implement' in proc.stderr: 126 test_case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 127 elif 'Can only provide HOST memory for this backend' in proc.stderr: 128 test_case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 129 elif 'Test not implemented in single precision' in proc.stderr: 130 test_case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 131 elif 'No SYCL devices of the requested type are available' in proc.stderr: 132 test_case.add_skipped_info('sycl device type not available {} {}'.format(test, ceed_resource)) 133 134 # check required failures 135 if not test_case.is_skipped(): 136 if test[:4] in 't006 t007'.split(): 137 check_required_failure(test_case, proc.stderr, 'No suitable backend:') 138 if test[:4] in 't008'.split(): 139 check_required_failure(test_case, proc.stderr, 'Available backend resources:') 140 if test[:4] in 't110 t111 t112 t113 t114'.split(): 141 check_required_failure(test_case, proc.stderr, 'Cannot grant CeedVector array access') 142 if test[:4] in 't115'.split(): 143 check_required_failure(test_case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 144 if test[:4] in 't116'.split(): 145 check_required_failure(test_case, proc.stderr, 'Cannot destroy CeedVector, the writable access lock is in use') 146 if test[:4] in 't117'.split(): 147 check_required_failure(test_case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 148 if test[:4] in 't118'.split(): 149 check_required_failure(test_case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 150 if test[:4] in 't215'.split(): 151 check_required_failure(test_case, proc.stderr, 'Cannot destroy CeedElemRestriction, a process has read access to the offset data') 152 if test[:4] in 't303'.split(): 153 check_required_failure(test_case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 154 if test[:4] in 't408'.split(): 155 check_required_failure(test_case, proc.stderr, 'CeedQFunctionContextGetData(): Cannot grant CeedQFunctionContext data access, a process has read access') 156 if test[:4] in 't409'.split() and contains_any(ceed_resource, ['memcheck']): 157 check_required_failure(test_case, proc.stderr, 'Context data changed while accessed in read-only mode') 158 159 # classify other results 160 if not test_case.is_skipped() and not test_case.status: 161 if proc.stderr: 162 test_case.add_failure_info('stderr', proc.stderr) 163 elif proc.returncode != 0: 164 test_case.add_error_info('returncode = {}'.format(proc.returncode)) 165 elif os.path.isfile(ref_stdout): 166 with open(ref_stdout) as ref: 167 diff = list(difflib.unified_diff(ref.readlines(), 168 proc.stdout.splitlines(keepends=True), 169 fromfile=ref_stdout, 170 tofile='New')) 171 if diff: 172 test_case.add_failure_info('stdout', output=''.join(diff)) 173 elif proc.stdout and test[:4] not in 't003': 174 test_case.add_failure_info('stdout', output=proc.stdout) 175 176 # store result 177 test_case.args = ' '.join(rargs) 178 test_cases.append(test_case) 179 180 if mode.lower() == "tap": 181 # print incremental output if TAP mode 182 print('# Test: {}'.format(test_case.name.split(' ')[1])) 183 print('# $ {}'.format(test_case.args)) 184 if test_case.is_error(): 185 print('not ok {} - ERROR: {}'.format(index, (test_case.errors[0]['message'] or "NO MESSAGE").strip())) 186 print('Output: \n{}'.format(test_case.errors[0]['output'].strip())) 187 if test_case.is_failure(): 188 print(' FAIL: {}'.format(index, (test_case.failures[0]['message'] or "NO MESSAGE").strip())) 189 print('Output: \n{}'.format(test_case.failures[0]['output'].strip())) 190 elif test_case.is_failure(): 191 print('not ok {} - FAIL: {}'.format(index, (test_case.failures[0]['message'] or "NO MESSAGE").strip())) 192 print('Output: \n{}'.format(test_case.failures[0]['output'].strip())) 193 elif test_case.is_skipped(): 194 print('ok {} - SKIP: {}'.format(index, (test_case.skipped[0]['message'] or "NO MESSAGE").strip())) 195 else: 196 print('ok {} - PASS'.format(index)) 197 sys.stdout.flush() 198 else: 199 # print error or failure information if JUNIT mode 200 if test_case.is_error() or test_case.is_failure(): 201 print('Test: {} {}'.format(test_case.name.split(' ')[0], test_case.name.split(' ')[1])) 202 print(' $ {}'.format(test_case.args)) 203 if test_case.is_error(): 204 print('ERROR: {}'.format((test_case.errors[0]['message'] or "NO MESSAGE").strip())) 205 print('Output: \n{}'.format((test_case.errors[0]['output'] or "NO OUTPUT").strip())) 206 if test_case.is_failure(): 207 print('FAIL: {}'.format((test_case.failures[0]['message'] or "NO MESSAGE").strip())) 208 print('Output: \n{}'.format((test_case.failures[0]['output'] or "NO OUTPUT").strip())) 209 sys.stdout.flush() 210 index += 1 211 212 return TestSuite(test, test_cases) 213 214if __name__ == '__main__': 215 import argparse 216 parser = argparse.ArgumentParser('Test runner with JUnit and TAP output') 217 parser.add_argument('--mode', help='Output mode, JUnit or TAP', default="JUnit") 218 parser.add_argument('--output', help='Output file to write test', default=None) 219 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 220 parser.add_argument('test', help='Test executable', nargs='?') 221 args = parser.parse_args() 222 223 if args.gather: 224 gather() 225 else: 226 backends = os.environ['BACKENDS'].split() 227 228 # run tests 229 result = run(args.test, backends, args.mode) 230 231 # build output 232 if args.mode.lower() == "junit": 233 junit_batch = '' 234 try: 235 junit_batch = '-' + os.environ['JUNIT_BATCH'] 236 except: 237 pass 238 output = (os.path.join('build', args.test + junit_batch + '.junit') 239 if args.output is None 240 else args.output) 241 242 with open(output, 'w') as fd: 243 TestSuite.to_file(fd, [result]) 244 elif args.mode.lower() != "tap": 245 raise Exception("output mode not recognized") 246 247 # check return code 248 for t in result.test_cases: 249 failures = len([c for c in result.test_cases if c.is_failure()]) 250 errors = len([c for c in result.test_cases if c.is_error()]) 251 if failures + errors > 0 and args.mode.lower() != "tap": 252 sys.exit(1) 253