1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([line.split()[1:] for line in open(file).readlines() 11 if line.startswith('//TESTARGS')], []) 12 elif os.path.splitext(file)[1] == '.usr': 13 return sum([line.split()[2:] for line in open(file).readlines() 14 if line.startswith('C TESTARGS')], []) 15 raise RuntimeError(f'Unrecognized extension for file: {file}') 16 17def get_source(test): 18 if test.startswith('petsc-'): 19 return os.path.join('examples', 'petsc', test[6:] + '.c') 20 elif test.startswith('mfem-'): 21 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 22 elif test.startswith('nek-'): 23 return os.path.join('examples', 'nek5000', test[4:] + '.usr') 24 elif test.startswith('ex'): 25 return os.path.join('examples', 'ceed', test + '.c') 26 27def get_testargs(test): 28 source = get_source(test) 29 if source is None: 30 return ['{ceed_resource}'] 31 return parse_testargs(source) 32 33def run(test, backends): 34 import subprocess 35 import time 36 import difflib 37 args = get_testargs(test) 38 testcases = [] 39 for ceed_resource in backends: 40 rargs = [os.path.join('build', test)] + args.copy() 41 rargs[rargs.index('{ceed_resource}')] = ceed_resource 42 start = time.time() 43 proc = subprocess.run(rargs, 44 stdout=subprocess.PIPE, 45 stderr=subprocess.PIPE, 46 encoding='utf-8') 47 48 case = TestCase(f'{test} {ceed_resource}', 49 elapsed_sec=time.time()-start, 50 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 51 stdout=proc.stdout, 52 stderr=proc.stderr) 53 ref_stdout = os.path.join('output', test + '.out') 54 if proc.stderr: 55 if 'OCCA backend failed to use' in proc.stderr: 56 case.add_skipped_info(f'occa mode not supported {test} {ceed_resource}') 57 elif 'Backend does not implement' in proc.stderr: 58 case.add_skipped_info(f'not implemented {test} {ceed_resource}') 59 elif 'access' in proc.stderr and test[:4] in 't103 t104 t105 t106 t107'.split(): 60 case.add_skipped_info(f'expected failure') 61 elif 'vectors incompatible' in proc.stderr and test[:4] in ['t308']: 62 case.add_skipped_info(f'expected failure') 63 else: 64 case.add_failure_info('stderr', proc.stderr) 65 if not case.is_skipped(): 66 if proc.returncode != 0: 67 case.add_error_info(f'returncode = {proc.returncode}') 68 elif os.path.isfile(ref_stdout): 69 with open(ref_stdout) as ref: 70 diff = list(difflib.unified_diff(ref.readlines(), 71 proc.stdout.splitlines(keepends=True), 72 fromfile=ref_stdout, 73 tofile='New')) 74 if diff: 75 case.add_failure_info('stdout', output=''.join(diff)) 76 elif proc.stdout: 77 case.add_failure_info('stdout', output=proc.stdout) 78 testcases.append(case) 79 return TestSuite(test, testcases) 80 81if __name__ == '__main__': 82 import argparse 83 parser = argparse.ArgumentParser('Test runner with JUnit output') 84 parser.add_argument('--output', help='Output file to write test', default=None) 85 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 86 parser.add_argument('test', help='Test executable', nargs='?') 87 args = parser.parse_args() 88 89 if args.gather: 90 gather() 91 else: 92 backends = os.environ['BACKENDS'].split() 93 94 result = run(args.test, backends) 95 output = (os.path.join('build', args.test + '.junit') 96 if args.output is None 97 else args.output) 98 with open(output, 'w') as fd: 99 TestSuite.to_file(fd, [result]) 100 101