1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([[[line.split()[1:], [line.split()[0].strip('//TESTARGS(name=').strip(')')]]] 11 for line in open(file).readlines() 12 if line.startswith('//TESTARGS')], []) 13 elif os.path.splitext(file)[1] == '.usr': 14 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 15 for line in open(file).readlines() 16 if line.startswith('C_TESTARGS')], []) 17 elif os.path.splitext(file)[1] in ['.f90']: 18 return sum([[[line.split()[1:], [line.split()[0].strip('C_TESTARGS(name=').strip(')')]]] 19 for line in open(file).readlines() 20 if line.startswith('! TESTARGS')], []) 21 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 22 23def get_source(test): 24 if test.startswith('petsc-'): 25 return os.path.join('examples', 'petsc', test[6:] + '.c') 26 elif test.startswith('mfem-'): 27 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 28 elif test.startswith('nek-'): 29 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 30 elif test.startswith('fluids-'): 31 return os.path.join('examples', 'fluids', test[7:] + '.c') 32 elif test.startswith('solids-'): 33 return os.path.join('examples', 'solids', test[7:] + '.c') 34 elif test.startswith('ex'): 35 return os.path.join('examples', 'ceed', test + '.c') 36 elif test.endswith('-f'): 37 return os.path.join('tests', test + '.f90') 38 else: 39 return os.path.join('tests', test + '.c') 40 41def get_testargs(source): 42 args = parse_testargs(source) 43 if not args: 44 return [(['{ceed_resource}'], [''])] 45 return args 46 47def check_required_failure(case, stderr, required): 48 if required in stderr: 49 case.status = 'fails with required: {}'.format(required) 50 else: 51 case.add_failure_info('required: {}'.format(required)) 52 53def contains_any(resource, substrings): 54 return any((sub in resource for sub in substrings)) 55 56def skip_rule(test, resource): 57 return any(( 58 test.startswith('fluids-') and contains_any(resource, ['occa']), 59 test.startswith('solids-') and contains_any(resource, ['occa']), 60 test.startswith('nek') and contains_any(resource, ['occa']), 61 test.startswith('t507') and contains_any(resource, ['occa']), 62 test.startswith('t318') and contains_any(resource, ['/gpu/cuda/ref']), 63 test.startswith('t506') and contains_any(resource, ['/gpu/cuda/shared']), 64 test.startswith('t568') and (contains_any(resource, ['/gpu/cuda/gen']) or resource.startswith('/gpu/hip')), 65 )) 66 67def run(test, backends): 68 import subprocess 69 import time 70 import difflib 71 source = get_source(test) 72 allargs = get_testargs(source) 73 74 testcases = [] 75 my_env = os.environ.copy() 76 my_env["CEED_ERROR_HANDLER"] = 'exit'; 77 for args, name in allargs: 78 for ceed_resource in backends: 79 rargs = [os.path.join('build', test)] + args.copy() 80 rargs[rargs.index('{ceed_resource}')] = ceed_resource 81 82 if skip_rule(test, ceed_resource): 83 case = TestCase('{} {}'.format(test, ceed_resource), 84 elapsed_sec=0, 85 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 86 stdout='', 87 stderr='') 88 case.add_skipped_info('Pre-run skip rule') 89 else: 90 start = time.time() 91 proc = subprocess.run(rargs, 92 stdout=subprocess.PIPE, 93 stderr=subprocess.PIPE, 94 env=my_env) 95 proc.stdout = proc.stdout.decode('utf-8') 96 proc.stderr = proc.stderr.decode('utf-8') 97 98 case = TestCase('{} {} {}'.format(test, *name, ceed_resource), 99 classname=os.path.dirname(source), 100 elapsed_sec=time.time()-start, 101 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 102 stdout=proc.stdout, 103 stderr=proc.stderr) 104 ref_stdout = os.path.join('tests/output', test + '.out') 105 106 if not case.is_skipped() and proc.stderr: 107 if 'OCCA backend failed to use' in proc.stderr: 108 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 109 elif 'Backend does not implement' in proc.stderr: 110 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 111 elif 'Can only provide HOST memory for this backend' in proc.stderr: 112 case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 113 elif 'Test not implemented in single precision' in proc.stderr: 114 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 115 116 if not case.is_skipped(): 117 if test[:4] in 't006 t007'.split(): 118 check_required_failure(case, proc.stderr, 'No suitable backend:') 119 if test[:4] in 't008'.split(): 120 check_required_failure(case, proc.stderr, 'Available backend resources:') 121 if test[:4] in 't110 t111 t112 t113 t114'.split(): 122 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector array access') 123 if test[:4] in 't115'.split(): 124 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 125 if test[:4] in 't116'.split(): 126 check_required_failure(case, proc.stderr, 'Cannot destroy CeedVector, the writable access lock is in use') 127 if test[:4] in 't117'.split(): 128 check_required_failure(case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 129 if test[:4] in 't118'.split(): 130 check_required_failure(case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 131 if test[:4] in 't215'.split(): 132 check_required_failure(case, proc.stderr, 'Cannot destroy CeedElemRestriction, a process has read access to the offset data') 133 if test[:4] in 't303'.split(): 134 check_required_failure(case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 135 if test[:4] in 't408'.split(): 136 check_required_failure(case, proc.stderr, 'CeedQFunctionContextGetData(): Cannot grant CeedQFunctionContext data access, a process has read access') 137 if test[:4] in 't409'.split() and contains_any(ceed_resource, ['memcheck']): 138 check_required_failure(case, proc.stderr, 'Context data changed while accessed in read-only mode') 139 140 if not case.is_skipped() and not case.status: 141 if proc.stderr: 142 case.add_failure_info('stderr', proc.stderr) 143 elif proc.returncode != 0: 144 case.add_error_info('returncode = {}'.format(proc.returncode)) 145 elif os.path.isfile(ref_stdout): 146 with open(ref_stdout) as ref: 147 diff = list(difflib.unified_diff(ref.readlines(), 148 proc.stdout.splitlines(keepends=True), 149 fromfile=ref_stdout, 150 tofile='New')) 151 if diff: 152 case.add_failure_info('stdout', output=''.join(diff)) 153 elif proc.stdout and test[:4] not in 't003': 154 case.add_failure_info('stdout', output=proc.stdout) 155 testcases.append(case) 156 return TestSuite(test, testcases) 157 158if __name__ == '__main__': 159 import argparse 160 parser = argparse.ArgumentParser('Test runner with JUnit output') 161 parser.add_argument('--output', help='Output file to write test', default=None) 162 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 163 parser.add_argument('test', help='Test executable', nargs='?') 164 args = parser.parse_args() 165 166 if args.gather: 167 gather() 168 else: 169 backends = os.environ['BACKENDS'].split() 170 171 result = run(args.test, backends) 172 173 junit_batch = '' 174 try: 175 junit_batch = '-' + os.environ['JUNIT_BATCH'] 176 except: 177 pass 178 output = (os.path.join('build', args.test + junit_batch + '.junit') 179 if args.output is None 180 else args.output) 181 182 with open(output, 'w') as fd: 183 TestSuite.to_file(fd, [result]) 184 185 for t in result.test_cases: 186 failures = len([c for c in result.test_cases if c.is_failure()]) 187 errors = len([c for c in result.test_cases if c.is_error()]) 188 if failures + errors > 0: 189 sys.exit(1) 190