1#!/usr/bin/env python3 2 3import os 4import sys 5sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6from junit_xml import TestCase, TestSuite 7 8def parse_testargs(file): 9 if os.path.splitext(file)[1] in ['.c', '.cpp']: 10 return sum([line.split()[1:] for line in open(file).readlines() 11 if line.startswith('//TESTARGS')], []) 12 elif os.path.splitext(file)[1] == '.usr': 13 return sum([line.split()[2:] for line in open(file).readlines() 14 if line.startswith('C TESTARGS')], []) 15 raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 16 17def get_source(test): 18 if test.startswith('petsc-'): 19 return os.path.join('examples', 'petsc', test[6:] + '.c') 20 elif test.startswith('mfem-'): 21 return os.path.join('examples', 'mfem', test[5:] + '.cpp') 22 elif test.startswith('nek-'): 23 return os.path.join('examples', 'nek5000', test[4:] + '.usr') 24 elif test.startswith('ex'): 25 return os.path.join('examples', 'ceed', test + '.c') 26 27def get_testargs(test): 28 source = get_source(test) 29 if source is None: 30 return ['{ceed_resource}'] 31 return parse_testargs(source) 32 33def check_required_failure(case, stderr, required): 34 if required in stderr: 35 case.status = 'fails with required: {}'.format(required) 36 else: 37 case.add_failure_info('required: {}'.format(required)) 38 39def run(test, backends): 40 import subprocess 41 import time 42 import difflib 43 args = get_testargs(test) 44 testcases = [] 45 for ceed_resource in backends: 46 rargs = [os.path.join('build', test)] + args.copy() 47 rargs[rargs.index('{ceed_resource}')] = ceed_resource 48 start = time.time() 49 proc = subprocess.run(rargs, 50 stdout=subprocess.PIPE, 51 stderr=subprocess.PIPE) 52 proc.stdout = proc.stdout.decode('utf-8') 53 proc.stderr = proc.stderr.decode('utf-8') 54 55 case = TestCase('{} {}'.format(test, ceed_resource), 56 elapsed_sec=time.time()-start, 57 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 58 stdout=proc.stdout, 59 stderr=proc.stderr) 60 ref_stdout = os.path.join('output', test + '.out') 61 62 if proc.stderr: 63 if 'OCCA backend failed to use' in proc.stderr: 64 case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 65 elif 'Backend does not implement' in proc.stderr: 66 case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 67 68 if not case.is_skipped(): 69 if test[:4] in 't103 t105 t106'.split(): 70 check_required_failure(case, proc.stderr, 'Cannot grant CeedVector array access') 71 if test[:4] in 't104'.split(): 72 check_required_failure(case, proc.stderr, 'Cannot destroy CeedVector, the access lock is in use') 73 if test[:4] in 't107'.split(): 74 check_required_failure(case, proc.stderr, 'Cannot restore CeedVector array access, access was not granted') 75 if test[:4] in 't308'.split(): 76 check_required_failure(case, proc.stderr, 'Length of input/output vectors incompatible with basis dimensions') 77 78 if not case.is_skipped() and not case.status: 79 if proc.stderr: 80 case.add_failure_info('stderr', proc.stderr) 81 elif proc.returncode != 0: 82 case.add_error_info('returncode = {}'.format(proc.returncode)) 83 elif os.path.isfile(ref_stdout): 84 with open(ref_stdout) as ref: 85 diff = list(difflib.unified_diff(ref.readlines(), 86 proc.stdout.splitlines(keepends=True), 87 fromfile=ref_stdout, 88 tofile='New')) 89 if diff: 90 case.add_failure_info('stdout', output=''.join(diff)) 91 elif proc.stdout: 92 case.add_failure_info('stdout', output=proc.stdout) 93 testcases.append(case) 94 return TestSuite(test, testcases) 95 96if __name__ == '__main__': 97 import argparse 98 parser = argparse.ArgumentParser('Test runner with JUnit output') 99 parser.add_argument('--output', help='Output file to write test', default=None) 100 parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 101 parser.add_argument('test', help='Test executable', nargs='?') 102 args = parser.parse_args() 103 104 if args.gather: 105 gather() 106 else: 107 backends = os.environ['BACKENDS'].split() 108 109 result = run(args.test, backends) 110 output = (os.path.join('build', args.test + '.junit') 111 if args.output is None 112 else args.output) 113 with open(output, 'w') as fd: 114 TestSuite.to_file(fd, [result]) 115 116