1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([[[line.split()[1:], [line.split()[0].strip('//TESTARGS(name=').strip(')')]]] 11 for line in open(file).readlines() 12 if line.startswith('//TESTARGS')], []) 13 elif os.path.splitext(file)[1] == '.usr': 14 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 15 for line in open(file).readlines() 16 if line.startswith('C_TESTARGS')], []) 17 elif os.path.splitext(file)[1] in ['.f90']: 18 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 19 for line in open(file).readlines() 20 if line.startswith('! TESTARGS')], []) 21 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 22 23def get_source(test): 24 if test.startswith('petsc-'): 25 return os.path.join('examples', 'petsc', test[6:] + '.c') 26 elif test.startswith('mfem-'): 27 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 28 elif test.startswith('nek-'): 29 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 30 elif test.startswith('fluids-'): 31 return os.path.join('examples', 'fluids', test[7:] + '.c') 32 elif test.startswith('solids-'): 33 return os.path.join('examples', 'solids', test[7:] + '.c') 34 elif test.startswith('ex'): 35 return os.path.join('examples', 'ceed', test + '.c') 36 elif test.endswith('-f'): 37 return os.path.join('tests', test + '.f90') 38 else: 39 return os.path.join('tests', test + '.c') 40 41def get_testargs(source): 42 args = parse_testargs(source) 43 if not args: 44 return [(['{ceed_resource}'], [''])] 45 return args 46 47def check_required_failure(case, stderr, required): 48 if required in stderr: 49 case.status = 'fails with required: {}'.format(required) 50 else: 51 case.add_failure_info('required: {}'.format(required)) 52 53def contains_any(resource, substrings): 54 return any((sub in resource for sub in substrings)) 55 56def skip_rule(test, resource): 57 return any(( 58 test.startswith('fluids-') and contains_any(resource, ['occa']), 59 test.startswith('solids-') and contains_any(resource, ['occa']), 60 test.startswith('nek') and contains_any(resource, ['occa']), 61 test.startswith('t507') and contains_any(resource, ['occa']), 62 test.startswith('t318') and contains_any(resource, ['/gpu/cuda/ref']), 63 test.startswith('t506') and contains_any(resource, ['/gpu/cuda/shared']), 64 )) 65 66def run(test, backends): 67 import subprocess 68 import time 69 import difflib 70 source = get_source(test) 71 allargs = get_testargs(source) 72 73 testcases = [] 74 my_env = os.environ.copy() 75 my_env["CEED_ERROR_HANDLER"] = 'exit'; 76 for args, name in allargs: 77 for ceed_resource in backends: 78 rargs = [os.path.join('build', test)] + args.copy() 79 rargs[rargs.index('{ceed_resource}')] = ceed_resource 80 81 if skip_rule(test, ceed_resource): 82 case = TestCase('{} {}'.format(test, ceed_resource), 83 elapsed_sec=0, 84 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 85 stdout='', 86 stderr='') 87 case.add_skipped_info('Pre-run skip rule') 88 else: 89 start = time.time() 90 proc = subprocess.run(rargs, 91 stdout=subprocess.PIPE, 92 stderr=subprocess.PIPE, 93 env=my_env) 94 proc.stdout = proc.stdout.decode('utf-8') 95 proc.stderr = proc.stderr.decode('utf-8') 96 97 case = TestCase('{} {} {}'.format(test, *name, ceed_resource), 98 classname=os.path.dirname(source), 99 elapsed_sec=time.time()-start, 100 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 101 stdout=proc.stdout, 102 stderr=proc.stderr) 103 ref_stdout = os.path.join('tests/output', test + '.out') 104 105 if not case.is_skipped() and proc.stderr: 106 if 'OCCA backend failed to use' in proc.stderr: 107 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 108 elif 'Backend does not implement' in proc.stderr: 109 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 110 elif 'Can only provide HOST memory for this backend' in proc.stderr: 111 case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 112 elif 'Test not implemented in single precision' in proc.stderr: 113 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 114 115 if not case.is_skipped(): 116 if test[:4] in 't006 t007'.split(): 117 check_required_failure(case, proc.stderr, 'No suitable backend:') 118 if test[:4] in 't008'.split(): 119 check_required_failure(case, proc.stderr, 'Available backend resources:') 120 if test[:4] in 't110 t111 t112 t113 t114'.split(): 121 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector array access') 122 if test[:4] in 't115'.split(): 123 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 124 if test[:4] in 't116'.split(): 125 check_required_failure(case, proc.stderr, 'Cannot destroy CeedVector, the writable access lock is in use') 126 if test[:4] in 't117'.split(): 127 check_required_failure(case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 128 if test[:4] in 't118'.split(): 129 check_required_failure(case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 130 if test[:4] in 't215'.split(): 131 check_required_failure(case, proc.stderr, 'Cannot destroy CeedElemRestriction, a process has read access to the offset data') 132 if test[:4] in 't303'.split(): 133 check_required_failure(case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 134 if test[:4] in 't408'.split(): 135 check_required_failure(case, proc.stderr, 'CeedQFunctionContextGetData(): Cannot grant CeedQFunctionContext data access, a process has read access') 136 137 if not case.is_skipped() and not case.status: 138 if proc.stderr: 139 case.add_failure_info('stderr', proc.stderr) 140 elif proc.returncode != 0: 141 case.add_error_info('returncode = {}'.format(proc.returncode)) 142 elif os.path.isfile(ref_stdout): 143 with open(ref_stdout) as ref: 144 diff = list(difflib.unified_diff(ref.readlines(), 145 proc.stdout.splitlines(keepends=True), 146 fromfile=ref_stdout, 147 tofile='New')) 148 if diff: 149 case.add_failure_info('stdout', output=''.join(diff)) 150 elif proc.stdout and test[:4] not in 't003': 151 case.add_failure_info('stdout', output=proc.stdout) 152 testcases.append(case) 153 return TestSuite(test, testcases) 154 155if __name__ == '__main__': 156 import argparse 157 parser = argparse.ArgumentParser('Test runner with JUnit output') 158 parser.add_argument('--output', help='Output file to write test', default=None) 159 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 160 parser.add_argument('test', help='Test executable', nargs='?') 161 args = parser.parse_args() 162 163 if args.gather: 164 gather() 165 else: 166 backends = os.environ['BACKENDS'].split() 167 168 result = run(args.test, backends) 169 170 junit_batch = '' 171 try: 172 junit_batch = '-' + os.environ['JUNIT_BATCH'] 173 except: 174 pass 175 output = (os.path.join('build', args.test + junit_batch + '.junit') 176 if args.output is None 177 else args.output) 178 179 with open(output, 'w') as fd: 180 TestSuite.to_file(fd, [result]) 181 182 for t in result.test_cases: 183 failures = len([c for c in result.test_cases if c.is_failure()]) 184 errors = len([c for c in result.test_cases if c.is_error()]) 185 if failures + errors > 0: 186 sys.exit(1) 187