1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([[line.split()[1:]] for line in open(file).readlines() 11 if line.startswith('//TESTARGS')], []) 12 elif os.path.splitext(file)[1] == '.usr': 13 return sum([[line.split()[2:]] for line in open(file).readlines() 14 if line.startswith('C TESTARGS')]) 15 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 16 17def get_source(test): 18 if test.startswith('petsc-'): 19 return os.path.join('examples', 'petsc', test[6:] + '.c') 20 elif test.startswith('mfem-'): 21 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 22 elif test.startswith('nek-'): 23 return os.path.join('examples', 'nek', 'bps', test[4:] + '.usr') 24 elif test.startswith('ns-'): 25 return os.path.join('examples', 'navier-stokes', test[3:] + '.c') 26 elif test.startswith('ex'): 27 return os.path.join('examples', 'ceed', test + '.c') 28 29def get_testargs(test): 30 source = get_source(test) 31 if source is None: 32 return [['{ceed_resource}']] 33 return parse_testargs(source) 34 35def check_required_failure(case, stderr, required): 36 if required in stderr: 37 case.status = 'fails with required: {}'.format(required) 38 else: 39 case.add_failure_info('required: {}'.format(required)) 40 41def contains_any(resource, substrings): 42 return any((sub in resource for sub in substrings)) 43 44def skip_rule(test, resource): 45 return any(( 46 test.startswith('ns-') and contains_any(resource, ['occa', 'gpu']), 47 test.startswith('petsc-multigrid') and contains_any(resource, ['occa']), 48 test.startswith('t506') and contains_any(resource, ['occa']), 49 test.startswith('t507') and contains_any(resource, ['occa']), 50 )) 51 52def run(test, backends): 53 import subprocess 54 import time 55 import difflib 56 allargs = get_testargs(test) 57 58 testcases = [] 59 for args in allargs: 60 for ceed_resource in backends: 61 rargs = [os.path.join('build', test)] + args.copy() 62 rargs[rargs.index('{ceed_resource}')] = ceed_resource 63 64 if skip_rule(test, ceed_resource): 65 case = TestCase('{} {}'.format(test, ceed_resource), 66 elapsed_sec=0, 67 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 68 stdout='', 69 stderr='') 70 case.add_skipped_info('Pre-run skip rule') 71 else: 72 start = time.time() 73 proc = subprocess.run(rargs, 74 stdout=subprocess.PIPE, 75 stderr=subprocess.PIPE) 76 proc.stdout = proc.stdout.decode('utf-8') 77 proc.stderr = proc.stderr.decode('utf-8') 78 79 case = TestCase('{} {}'.format(test, ceed_resource), 80 elapsed_sec=time.time()-start, 81 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 82 stdout=proc.stdout, 83 stderr=proc.stderr) 84 ref_stdout = os.path.join('tests/output', test + '.out') 85 86 if not case.is_skipped() and proc.stderr: 87 if 'OCCA backend failed to use' in proc.stderr: 88 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 89 elif 'Backend does not implement' in proc.stderr: 90 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 91 elif 'Can only provide to HOST memory' in proc.stderr: 92 case.add_skipped_info('device memory not supported {} {}'.format(test, ceed_resource)) 93 94 if not case.is_skipped(): 95 if test[:4] in 't110 t111 t112 t113 t114'.split(): 96 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector array access') 97 if test[:4] in 't115'.split(): 98 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector read-only array access, the access lock is already in use') 99 if test[:4] in 't116'.split(): 100 check_required_failure(case, proc.stderr, 'Cannot destroy CeedVector, the access lock is in use') 101 if test[:4] in 't117'.split(): 102 check_required_failure(case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 103 if test[:4] in 't118'.split(): 104 check_required_failure(case, proc.stderr, 'Cannot sync CeedVector, the access lock is already in use') 105 if test[:4] in 't303'.split(): 106 check_required_failure(case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 107 108 if not case.is_skipped() and not case.status: 109 if proc.stderr: 110 case.add_failure_info('stderr', proc.stderr) 111 elif proc.returncode != 0: 112 case.add_error_info('returncode = {}'.format(proc.returncode)) 113 elif os.path.isfile(ref_stdout): 114 with open(ref_stdout) as ref: 115 diff = list(difflib.unified_diff(ref.readlines(), 116 proc.stdout.splitlines(keepends=True), 117 fromfile=ref_stdout, 118 tofile='New')) 119 if diff: 120 case.add_failure_info('stdout', output=''.join(diff)) 121 elif proc.stdout: 122 case.add_failure_info('stdout', output=proc.stdout) 123 testcases.append(case) 124 return TestSuite(test, testcases) 125 126if __name__ == '__main__': 127 import argparse 128 parser = argparse.ArgumentParser('Test runner with JUnit output') 129 parser.add_argument('--output', help='Output file to write test', default=None) 130 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 131 parser.add_argument('test', help='Test executable', nargs='?') 132 args = parser.parse_args() 133 134 if args.gather: 135 gather() 136 else: 137 backends = os.environ['BACKENDS'].split() 138 139 result = run(args.test, backends) 140 output = (os.path.join('build', args.test + '.junit') 141 if args.output is None 142 else args.output) 143 with open(output, 'w') as fd: 144 TestSuite.to_file(fd, [result]) 145 146