1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([[[line.split()[1:], [line.split()[0].strip('//TESTARGS(name=').strip(')')]]] 11 for line in open(file).readlines() 12 if line.startswith('//TESTARGS')], []) 13 elif os.path.splitext(file)[1] == '.usr': 14 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 15 for line in open(file).readlines() 16 if line.startswith('C_TESTARGS')], []) 17 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 18 19def get_source(test): 20 if test.startswith('petsc-'): 21 return os.path.join('examples', 'petsc', test[6:] + '.c') 22 elif test.startswith('mfem-'): 23 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 24 elif test.startswith('nek-'): 25 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 26 elif test.startswith('fluids-'): 27 return os.path.join('examples', 'fluids', test[7:] + '.c') 28 elif test.startswith('solids-'): 29 return os.path.join('examples', 'solids', test[7:] + '.c') 30 elif test.startswith('ex'): 31 return os.path.join('examples', 'ceed', test + '.c') 32 33def get_testargs(test): 34 source = get_source(test) 35 if source is None: 36 return [[['{ceed_resource}'], ['']]] 37 return parse_testargs(source) 38 39def check_required_failure(case, stderr, required): 40 if required in stderr: 41 case.status = 'fails with required: {}'.format(required) 42 else: 43 case.add_failure_info('required: {}'.format(required)) 44 45def contains_any(resource, substrings): 46 return any((sub in resource for sub in substrings)) 47 48def skip_rule(test, resource): 49 return any(( 50 test.startswith('fluids-') and contains_any(resource, ['occa', 'magma']), 51 test.startswith('solids-') and contains_any(resource, ['occa']), 52 test.startswith('nek') and contains_any(resource, ['occa']), 53 test.startswith('t507') and contains_any(resource, ['occa']), 54 test.startswith('t318') and contains_any(resource, ['magma', '/gpu/cuda/ref']), 55 test.startswith('t506') and contains_any(resource, ['magma', '/gpu/cuda/shared']), 56 )) 57 58def run(test, backends): 59 import subprocess 60 import time 61 import difflib 62 allargs = get_testargs(test) 63 64 testcases = [] 65 my_env = os.environ.copy() 66 my_env["CEED_ERROR_HANDLER"] = 'exit'; 67 for args, name in allargs: 68 for ceed_resource in backends: 69 rargs = [os.path.join('build', test)] + args.copy() 70 rargs[rargs.index('{ceed_resource}')] = ceed_resource 71 72 if skip_rule(test, ceed_resource): 73 case = TestCase('{} {}'.format(test, ceed_resource), 74 elapsed_sec=0, 75 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 76 stdout='', 77 stderr='') 78 case.add_skipped_info('Pre-run skip rule') 79 else: 80 start = time.time() 81 proc = subprocess.run(rargs, 82 stdout=subprocess.PIPE, 83 stderr=subprocess.PIPE, 84 env=my_env) 85 proc.stdout = proc.stdout.decode('utf-8') 86 proc.stderr = proc.stderr.decode('utf-8') 87 88 case = TestCase('{} {} {}'.format(test, *name, ceed_resource), 89 elapsed_sec=time.time()-start, 90 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 91 stdout=proc.stdout, 92 stderr=proc.stderr) 93 ref_stdout = os.path.join('tests/output', test + '.out') 94 95 if not case.is_skipped() and proc.stderr: 96 if 'OCCA backend failed to use' in proc.stderr: 97 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 98 elif 'Backend does not implement' in proc.stderr: 99 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 100 elif 'Can only provide to HOST memory' in proc.stderr: 101 case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 102 103 if not case.is_skipped(): 104 if test[:4] in 't006 t007'.split(): 105 check_required_failure(case, proc.stderr, 'No suitable backend:') 106 if test[:4] in 't008'.split(): 107 check_required_failure(case, proc.stderr, 'Available backend resources:') 108 if test[:4] in 't110 t111 t112 t113 t114'.split(): 109 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector array access') 110 if test[:4] in 't115'.split(): 111 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 112 if test[:4] in 't116'.split(): 113 check_required_failure(case, proc.stderr, 'Cannot destroy CeedVector, the writable access lock is in use') 114 if test[:4] in 't117'.split(): 115 check_required_failure(case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 116 if test[:4] in 't118'.split(): 117 check_required_failure(case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 118 if test[:4] in 't215'.split(): 119 check_required_failure(case, proc.stderr, 'Cannot destroy CeedElemRestriction, a process has read access to the offset data') 120 if test[:4] in 't303'.split(): 121 check_required_failure(case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 122 123 if not case.is_skipped() and not case.status: 124 if proc.stderr: 125 case.add_failure_info('stderr', proc.stderr) 126 elif proc.returncode != 0: 127 case.add_error_info('returncode = {}'.format(proc.returncode)) 128 elif os.path.isfile(ref_stdout): 129 with open(ref_stdout) as ref: 130 diff = list(difflib.unified_diff(ref.readlines(), 131 proc.stdout.splitlines(keepends=True), 132 fromfile=ref_stdout, 133 tofile='New')) 134 if diff: 135 case.add_failure_info('stdout', output=''.join(diff)) 136 elif proc.stdout and test[:4] not in 't003': 137 case.add_failure_info('stdout', output=proc.stdout) 138 testcases.append(case) 139 return TestSuite(test, testcases) 140 141if __name__ == '__main__': 142 import argparse 143 parser = argparse.ArgumentParser('Test runner with JUnit output') 144 parser.add_argument('--output', help='Output file to write test', default=None) 145 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 146 parser.add_argument('test', help='Test executable', nargs='?') 147 args = parser.parse_args() 148 149 if args.gather: 150 gather() 151 else: 152 backends = os.environ['BACKENDS'].split() 153 154 result = run(args.test, backends) 155 output = (os.path.join('build', args.test + '.junit') 156 if args.output is None 157 else args.output) 158 with open(output, 'w') as fd: 159 TestSuite.to_file(fd, [result]) 160 for t in result.test_cases: 161 failures = len([c for c in result.test_cases if c.is_failure()]) 162 errors = len([c for c in result.test_cases if c.is_error()]) 163 if failures + errors > 0: 164 sys.exit(1) 165