1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8 9def parse_testargs(file): 10 if os.path.splitext(file)[1] in ['.c', '.cpp']: 11 return sum([[[line.split()[1:], [line.split()[0].strip('//TESTARGS(name=').strip(')')]]] 12 for line in open(file).readlines() 13 if line.startswith('//TESTARGS')], []) 14 elif os.path.splitext(file)[1] == '.usr': 15 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 16 for line in open(file).readlines() 17 if line.startswith('C_TESTARGS')], []) 18 elif os.path.splitext(file)[1] in ['.f90']: 19 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 20 for line in open(file).readlines() 21 if line.startswith('! TESTARGS')], []) 22 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 23 24 25def get_source(test): 26 if test.startswith('petsc-'): 27 return os.path.join('examples', 'petsc', test[6:] + '.c') 28 elif test.startswith('mfem-'): 29 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 30 elif test.startswith('nek-'): 31 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 32 elif test.startswith('fluids-'): 33 return os.path.join('examples', 'fluids', test[7:] + '.c') 34 elif test.startswith('solids-'): 35 return os.path.join('examples', 'solids', test[7:] + '.c') 36 elif test.startswith('ex'): 37 return os.path.join('examples', 'ceed', test + '.c') 38 elif test.endswith('-f'): 39 return os.path.join('tests', test + '.f90') 40 else: 41 return os.path.join('tests', test + '.c') 42 43 44def get_testargs(source): 45 args = parse_testargs(source) 46 if not args: 47 return [(['{ceed_resource}'], [''])] 48 return args 49 50 51def check_required_failure(case, stderr, required): 52 if required in stderr: 53 case.status = 'fails with required: {}'.format(required) 54 else: 55 case.add_failure_info('required: {}'.format(required)) 56 57 58def contains_any(resource, substrings): 59 return any((sub in resource for sub in substrings)) 60 61 62def skip_rule(test, resource): 63 return any(( 64 test.startswith('fluids-') and contains_any(resource, ['occa']), 65 test.startswith('solids-') and contains_any(resource, ['occa']), 66 test.startswith('nek') and contains_any(resource, ['occa']), 67 test.startswith('t507') and contains_any(resource, ['occa']), 68 test.startswith('t318') and contains_any(resource, ['/gpu/cuda/ref']), 69 test.startswith('t506') and contains_any(resource, ['/gpu/cuda/shared']), 70 )) 71 72 73def run(test, backends): 74 import subprocess 75 import time 76 import difflib 77 source = get_source(test) 78 all_args = get_testargs(source) 79 80 test_cases = [] 81 my_env = os.environ.copy() 82 my_env["CEED_ERROR_HANDLER"] = 'exit' 83 for args, name in all_args: 84 for ceed_resource in backends: 85 rargs = [os.path.join('build', test)] + args.copy() 86 rargs[rargs.index('{ceed_resource}')] = ceed_resource 87 88 if skip_rule(test, ceed_resource): 89 case = TestCase('{} {}'.format(test, ceed_resource), 90 elapsed_sec=0, 91 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 92 stdout='', 93 stderr='') 94 case.add_skipped_info('Pre-run skip rule') 95 else: 96 start = time.time() 97 proc = subprocess.run(rargs, 98 stdout=subprocess.PIPE, 99 stderr=subprocess.PIPE, 100 env=my_env) 101 proc.stdout = proc.stdout.decode('utf-8') 102 proc.stderr = proc.stderr.decode('utf-8') 103 104 case = TestCase('{} {} {}'.format(test, *name, ceed_resource), 105 classname=os.path.dirname(source), 106 elapsed_sec=time.time() - start, 107 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 108 stdout=proc.stdout, 109 stderr=proc.stderr) 110 ref_stdout = os.path.join('tests/output', test + '.out') 111 112 if not case.is_skipped() and proc.stderr: 113 if 'OCCA backend failed to use' in proc.stderr: 114 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 115 elif 'Backend does not implement' in proc.stderr: 116 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 117 elif 'Can only provide HOST memory for this backend' in proc.stderr: 118 case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 119 elif 'Test not implemented in single precision' in proc.stderr: 120 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 121 122 if not case.is_skipped(): 123 if test[:4] in 't006 t007'.split(): 124 check_required_failure(case, proc.stderr, 'No suitable backend:') 125 if test[:4] in 't008'.split(): 126 check_required_failure(case, proc.stderr, 'Available backend resources:') 127 if test[:4] in 't110 t111 t112 t113 t114'.split(): 128 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector array access') 129 if test[:4] in 't115'.split(): 130 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 131 if test[:4] in 't116'.split(): 132 check_required_failure(case, proc.stderr, 'Cannot destroy CeedVector, the writable access lock is in use') 133 if test[:4] in 't117'.split(): 134 check_required_failure(case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 135 if test[:4] in 't118'.split(): 136 check_required_failure(case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 137 if test[:4] in 't215'.split(): 138 check_required_failure(case, proc.stderr, 'Cannot destroy CeedElemRestriction, a process has read access to the offset data') 139 if test[:4] in 't303'.split(): 140 check_required_failure(case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 141 if test[:4] in 't408'.split(): 142 check_required_failure(case, proc.stderr, 'CeedQFunctionContextGetData(): Cannot grant CeedQFunctionContext data access, a process has read access') 143 if test[:4] in 't409'.split() and contains_any(ceed_resource, ['memcheck']): 144 check_required_failure(case, proc.stderr, 'Context data changed while accessed in read-only mode') 145 146 if not case.is_skipped() and not case.status: 147 if proc.stderr: 148 case.add_failure_info('stderr', proc.stderr) 149 elif proc.returncode != 0: 150 case.add_error_info('returncode = {}'.format(proc.returncode)) 151 elif os.path.isfile(ref_stdout): 152 with open(ref_stdout) as ref: 153 diff = list(difflib.unified_diff(ref.readlines(), 154 proc.stdout.splitlines(keepends=True), 155 fromfile=ref_stdout, 156 tofile='New')) 157 if diff: 158 case.add_failure_info('stdout', output=''.join(diff)) 159 elif proc.stdout and test[:4] not in 't003': 160 case.add_failure_info('stdout', output=proc.stdout) 161 case.args = ' '.join(rargs) 162 test_cases.append(case) 163 return TestSuite(test, test_cases) 164 165if __name__ == '__main__': 166 import argparse 167 parser = argparse.ArgumentParser('Test runner with JUnit and TAP output') 168 parser.add_argument('--mode', help='Output mode, JUnit or TAP', default="JUnit") 169 parser.add_argument('--output', help='Output file to write test', default=None) 170 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 171 parser.add_argument('test', help='Test executable', nargs='?') 172 args = parser.parse_args() 173 174 if args.gather: 175 gather() 176 else: 177 backends = os.environ['BACKENDS'].split() 178 179 result = run(args.test, backends) 180 181 if args.mode.lower() == "junit": 182 junit_batch = '' 183 try: 184 junit_batch = '-' + os.environ['JUNIT_BATCH'] 185 except: 186 pass 187 output = (os.path.join('build', args.test + junit_batch + '.junit') 188 if args.output is None 189 else args.output) 190 191 with open(output, 'w') as fd: 192 TestSuite.to_file(fd, [result]) 193 elif args.mode.lower() == "tap": 194 print('1..' + str(len(result.test_cases))) 195 for i in range(len(result.test_cases)): 196 test_case = result.test_cases[i] 197 test_index = str(i + 1) 198 print('# Test: ' + test_case.name.split(' ')[1]) 199 print('# $ ' + test_case.args) 200 if test_case.is_error(): 201 message = test_case.is_error() if isinstance(test_case.is_error(), str) else test_case.errors[0]['message'] 202 print('not ok {} - ERROR: {}'.format(test_index, message.strip())) 203 print(test_case.errors[0]['output'].strip()) 204 elif test_case.is_failure(): 205 message = test_case.is_failure() if isinstance(test_case.is_failure(), str) else test_case.failures[0]['message'] 206 print('not ok {} - FAIL: {}'.format(test_index, message.strip())) 207 print(test_case.failures[0]['output'].strip()) 208 elif test_case.is_skipped(): 209 message = test_case.is_skipped() if isinstance(test_case.is_skipped(), str) else test_case.skipped[0]['message'] 210 print('ok {} - SKIP: {}'.format(test_index, message.strip())) 211 else: 212 print('ok {} - PASS'.format(test_index)) 213 else: 214 raise Exception("output mode not recognized") 215 216 for t in result.test_cases: 217 failures = len([c for c in result.test_cases if c.is_failure()]) 218 errors = len([c for c in result.test_cases if c.is_error()]) 219 if failures + errors > 0: 220 sys.exit(1) 221