18ec9d54bSJed Brown#!/usr/bin/env python3 28ec9d54bSJed Brown 38ec9d54bSJed Brownimport os 48ec9d54bSJed Brownimport sys 58ec9d54bSJed Brownsys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 68ec9d54bSJed Brownfrom junit_xml import TestCase, TestSuite 78ec9d54bSJed Brown 88ec9d54bSJed Browndef parse_testargs(file): 98ec9d54bSJed Brown if os.path.splitext(file)[1] in ['.c', '.cpp']: 108ec9d54bSJed Brown return sum([line.split()[1:] for line in open(file).readlines() 118ec9d54bSJed Brown if line.startswith('//TESTARGS')], []) 128ec9d54bSJed Brown elif os.path.splitext(file)[1] == '.usr': 138ec9d54bSJed Brown return sum([line.split()[2:] for line in open(file).readlines() 148ec9d54bSJed Brown if line.startswith('C TESTARGS')], []) 15*9bcbe8bdSJed Brown raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 168ec9d54bSJed Brown 178ec9d54bSJed Browndef get_source(test): 188ec9d54bSJed Brown if test.startswith('petsc-'): 198ec9d54bSJed Brown return os.path.join('examples', 'petsc', test[6:] + '.c') 208ec9d54bSJed Brown elif test.startswith('mfem-'): 218ec9d54bSJed Brown return os.path.join('examples', 'mfem', test[5:] + '.cpp') 228ec9d54bSJed Brown elif test.startswith('nek-'): 238ec9d54bSJed Brown return os.path.join('examples', 'nek5000', test[4:] + '.usr') 248ec9d54bSJed Brown elif test.startswith('ex'): 258ec9d54bSJed Brown return os.path.join('examples', 'ceed', test + '.c') 268ec9d54bSJed Brown 278ec9d54bSJed Browndef get_testargs(test): 288ec9d54bSJed Brown source = get_source(test) 298ec9d54bSJed Brown if source is None: 308ec9d54bSJed Brown return ['{ceed_resource}'] 318ec9d54bSJed Brown return parse_testargs(source) 328ec9d54bSJed Brown 338ec9d54bSJed Browndef run(test, backends): 348ec9d54bSJed Brown import subprocess 358ec9d54bSJed Brown import time 368ec9d54bSJed Brown import difflib 378ec9d54bSJed Brown args = get_testargs(test) 388ec9d54bSJed Brown testcases = [] 398ec9d54bSJed Brown for ceed_resource in backends: 408ec9d54bSJed Brown rargs = [os.path.join('build', test)] + args.copy() 418ec9d54bSJed Brown rargs[rargs.index('{ceed_resource}')] = ceed_resource 428ec9d54bSJed Brown start = time.time() 438ec9d54bSJed Brown proc = subprocess.run(rargs, 448ec9d54bSJed Brown stdout=subprocess.PIPE, 458ec9d54bSJed Brown stderr=subprocess.PIPE, 468ec9d54bSJed Brown encoding='utf-8') 478ec9d54bSJed Brown 48*9bcbe8bdSJed Brown case = TestCase('{} {}'.format(test, ceed_resource), 498ec9d54bSJed Brown elapsed_sec=time.time()-start, 508ec9d54bSJed Brown timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 518ec9d54bSJed Brown stdout=proc.stdout, 528ec9d54bSJed Brown stderr=proc.stderr) 538ec9d54bSJed Brown ref_stdout = os.path.join('output', test + '.out') 548ec9d54bSJed Brown if proc.stderr: 558ec9d54bSJed Brown if 'OCCA backend failed to use' in proc.stderr: 56*9bcbe8bdSJed Brown case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 578ec9d54bSJed Brown elif 'Backend does not implement' in proc.stderr: 58*9bcbe8bdSJed Brown case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 598ec9d54bSJed Brown elif 'access' in proc.stderr and test[:4] in 't103 t104 t105 t106 t107'.split(): 60*9bcbe8bdSJed Brown case.add_skipped_info('expected failure') 618ec9d54bSJed Brown elif 'vectors incompatible' in proc.stderr and test[:4] in ['t308']: 62*9bcbe8bdSJed Brown case.add_skipped_info('expected failure') 638ec9d54bSJed Brown else: 648ec9d54bSJed Brown case.add_failure_info('stderr', proc.stderr) 658ec9d54bSJed Brown if not case.is_skipped(): 668ec9d54bSJed Brown if proc.returncode != 0: 67*9bcbe8bdSJed Brown case.add_error_info('returncode = {}'.format(proc.returncode)) 688ec9d54bSJed Brown elif os.path.isfile(ref_stdout): 698ec9d54bSJed Brown with open(ref_stdout) as ref: 708ec9d54bSJed Brown diff = list(difflib.unified_diff(ref.readlines(), 718ec9d54bSJed Brown proc.stdout.splitlines(keepends=True), 728ec9d54bSJed Brown fromfile=ref_stdout, 738ec9d54bSJed Brown tofile='New')) 748ec9d54bSJed Brown if diff: 758ec9d54bSJed Brown case.add_failure_info('stdout', output=''.join(diff)) 768ec9d54bSJed Brown elif proc.stdout: 778ec9d54bSJed Brown case.add_failure_info('stdout', output=proc.stdout) 788ec9d54bSJed Brown testcases.append(case) 798ec9d54bSJed Brown return TestSuite(test, testcases) 808ec9d54bSJed Brown 818ec9d54bSJed Brownif __name__ == '__main__': 828ec9d54bSJed Brown import argparse 838ec9d54bSJed Brown parser = argparse.ArgumentParser('Test runner with JUnit output') 848ec9d54bSJed Brown parser.add_argument('--output', help='Output file to write test', default=None) 858ec9d54bSJed Brown parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 868ec9d54bSJed Brown parser.add_argument('test', help='Test executable', nargs='?') 878ec9d54bSJed Brown args = parser.parse_args() 888ec9d54bSJed Brown 898ec9d54bSJed Brown if args.gather: 908ec9d54bSJed Brown gather() 918ec9d54bSJed Brown else: 928ec9d54bSJed Brown backends = os.environ['BACKENDS'].split() 938ec9d54bSJed Brown 948ec9d54bSJed Brown result = run(args.test, backends) 958ec9d54bSJed Brown output = (os.path.join('build', args.test + '.junit') 968ec9d54bSJed Brown if args.output is None 978ec9d54bSJed Brown else args.output) 988ec9d54bSJed Brown with open(output, 'w') as fd: 998ec9d54bSJed Brown TestSuite.to_file(fd, [result]) 1008ec9d54bSJed Brown 101