1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([line.split()[1:] for line in open(file).readlines() 11 if line.startswith('//TESTARGS')], []) 12 elif os.path.splitext(file)[1] == '.usr': 13 return sum([line.split()[2:] for line in open(file).readlines() 14 if line.startswith('C TESTARGS')], []) 15 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 16 17def get_source(test): 18 if test.startswith('petsc-'): 19 return os.path.join('examples', 'petsc', test[6:] + '.c') 20 elif test.startswith('mfem-'): 21 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 22 elif test.startswith('nek-'): 23 return os.path.join('examples', 'nek5000', test[4:] + '.usr') 24 elif test.startswith('ex'): 25 return os.path.join('examples', 'ceed', test + '.c') 26 27def get_testargs(test): 28 source = get_source(test) 29 if source is None: 30 return ['{ceed_resource}'] 31 return parse_testargs(source) 32 33def run(test, backends): 34 import subprocess 35 import time 36 import difflib 37 args = get_testargs(test) 38 testcases = [] 39 for ceed_resource in backends: 40 rargs = [os.path.join('build', test)] + args.copy() 41 rargs[rargs.index('{ceed_resource}')] = ceed_resource 42 start = time.time() 43 proc = subprocess.run(rargs, 44 stdout=subprocess.PIPE, 45 stderr=subprocess.PIPE) 46 proc.stdout = proc.stdout.decode('utf-8') 47 proc.stderr = proc.stderr.decode('utf-8') 48 49 case = TestCase('{} {}'.format(test, ceed_resource), 50 elapsed_sec=time.time()-start, 51 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 52 stdout=proc.stdout, 53 stderr=proc.stderr) 54 ref_stdout = os.path.join('output', test + '.out') 55 if proc.stderr: 56 if 'OCCA backend failed to use' in proc.stderr: 57 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 58 elif 'Backend does not implement' in proc.stderr: 59 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 60 elif 'access' in proc.stderr and test[:4] in 't103 t104 t105 t106 t107'.split(): 61 case.add_skipped_info('expected failure') 62 elif 'vectors incompatible' in proc.stderr and test[:4] in ['t308']: 63 case.add_skipped_info('expected failure') 64 else: 65 case.add_failure_info('stderr', proc.stderr) 66 if not case.is_skipped(): 67 if proc.returncode != 0: 68 case.add_error_info('returncode = {}'.format(proc.returncode)) 69 elif os.path.isfile(ref_stdout): 70 with open(ref_stdout) as ref: 71 diff = list(difflib.unified_diff(ref.readlines(), 72 proc.stdout.splitlines(keepends=True), 73 fromfile=ref_stdout, 74 tofile='New')) 75 if diff: 76 case.add_failure_info('stdout', output=''.join(diff)) 77 elif proc.stdout: 78 case.add_failure_info('stdout', output=proc.stdout) 79 testcases.append(case) 80 return TestSuite(test, testcases) 81 82if __name__ == '__main__': 83 import argparse 84 parser = argparse.ArgumentParser('Test runner with JUnit output') 85 parser.add_argument('--output', help='Output file to write test', default=None) 86 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 87 parser.add_argument('test', help='Test executable', nargs='?') 88 args = parser.parse_args() 89 90 if args.gather: 91 gather() 92 else: 93 backends = os.environ['BACKENDS'].split() 94 95 result = run(args.test, backends) 96 output = (os.path.join('build', args.test + '.junit') 97 if args.output is None 98 else args.output) 99 with open(output, 'w') as fd: 100 TestSuite.to_file(fd, [result]) 101 102