1#!/usr/bin/env python3 2from junit_common import * 3 4 5def create_argparser() -> argparse.ArgumentParser: 6 """Creates argument parser to read command line arguments 7 8 Returns: 9 argparse.ArgumentParser: Created `ArgumentParser` 10 """ 11 parser = argparse.ArgumentParser('Test runner with JUnit and TAP output') 12 parser.add_argument( 13 '-c', 14 '--ceed-backends', 15 type=str, 16 nargs='*', 17 default=['/cpu/self'], 18 help='libCEED backend to use with convergence tests') 19 parser.add_argument( 20 '-m', 21 '--mode', 22 type=RunMode, 23 action=CaseInsensitiveEnumAction, 24 help='Output mode, junit or tap', 25 default=RunMode.JUNIT) 26 parser.add_argument('-n', '--nproc', type=int, default=1, help='number of MPI processes') 27 parser.add_argument('-o', '--output', type=Optional[Path], default=None, help='Output file to write test') 28 parser.add_argument('-b', '--junit-batch', type=str, default='', help='Name of JUnit batch for output file') 29 parser.add_argument('-np', '--pool-size', type=int, default=1, help='Number of test cases to run in parallel') 30 parser.add_argument('-s', '--smartredis_dir', type=str, default='', help='path to SmartSim library, if present') 31 parser.add_argument('--has_torch', type=bool, default=False, help='Whether to build with torch') 32 parser.add_argument('test', help='Test executable', nargs='?') 33 34 return parser 35 36 37# Necessary functions for running tests 38class CeedSuiteSpec(SuiteSpec): 39 def __init__(self, has_torch: bool): 40 self.has_torch: bool = has_torch 41 42 def get_source_path(self, test: str) -> Path: 43 """Compute path to test source file 44 45 Args: 46 test (str): Name of test 47 48 Returns: 49 Path: Path to source file 50 """ 51 return Path(test).with_suffix('.c') 52 53 # get path to executable 54 def get_run_path(self, test: str) -> Path: 55 """Compute path to built test executable file 56 57 Args: 58 test (str): Name of test 59 60 Returns: 61 Path: Path to test executable 62 """ 63 return Path('build') / test 64 65 def get_output_path(self, test: str, output_file: str) -> Path: 66 """Compute path to expected output file 67 68 Args: 69 test (str): Name of test 70 output_file (str): File name of output file 71 72 Returns: 73 Path: Path to expected output file 74 """ 75 return Path('tests') / 'output' / output_file 76 77 def check_pre_skip(self, test: str, spec: TestSpec, resource: str, nproc: int) -> Optional[str]: 78 """Check if a test case should be skipped prior to running, returning the reason for skipping 79 80 Args: 81 test (str): Name of test 82 spec (TestSpec): Test case specification 83 resource (str): libCEED backend 84 nproc (int): Number of MPI processes to use when running test case 85 86 Returns: 87 Optional[str]: Skip reason, or `None` if test case should not be skipped 88 """ 89 if contains_any(resource, ['occa']) and startswith_any( 90 test, ['t4', 't5', 'ex', 'mfem', 'nek', 'petsc', 'fluids', 'solids']): 91 return 'OCCA mode not supported' 92 if test.startswith('t318') and contains_any(resource, ['/gpu/cuda/ref']): 93 return 'CUDA ref backend not supported' 94 if test.startswith('t506') and contains_any(resource, ['/gpu/cuda/shared']): 95 return 'CUDA shared backend not supported' 96 for condition in spec.only: 97 if (condition == 'cpu') and ('gpu' in resource): 98 return 'CPU only test with GPU backend' 99 if condition == 'torch' and not self.has_torch: 100 return 'PyTorch only test without USE_TORCH=1' 101 102 def check_post_skip(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Optional[str]: 103 """Check if a test case should be allowed to fail, based on its stderr output 104 105 Args: 106 test (str): Name of test 107 spec (TestSpec): Test case specification 108 resource (str): libCEED backend 109 stderr (str): Standard error output from test case execution 110 111 Returns: 112 Optional[str]: Skip reason, or `None` if unexpeced error 113 """ 114 if 'OCCA backend failed to use' in stderr: 115 return f'OCCA mode not supported' 116 elif 'Backend does not implement' in stderr: 117 return f'Backend does not implement' 118 elif 'Can only provide HOST memory for this backend' in stderr: 119 return f'Device memory not supported' 120 elif 'Can only set HOST memory for this backend' in stderr: 121 return f'Device memory not supported' 122 elif 'Test not implemented in single precision' in stderr: 123 return f'Test not implemented in single precision' 124 elif 'No SYCL devices of the requested type are available' in stderr: 125 return f'SYCL device type not available' 126 elif 'You may need to add --download-ctetgen or --download-tetgen' in stderr: 127 return f'Tet mesh generator not installed for {test}, {spec.name}' 128 return None 129 130 def check_required_failure(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Tuple[str, bool]: 131 """Check whether a test case is expected to fail and if it failed expectedly 132 133 Args: 134 test (str): Name of test 135 spec (TestSpec): Test case specification 136 resource (str): libCEED backend 137 stderr (str): Standard error output from test case execution 138 139 Returns: 140 tuple[str, bool]: Tuple of the expected failure string and whether it was present in `stderr` 141 """ 142 test_id: str = test[:4] 143 fail_str: str = '' 144 if test_id in ['t006', 't007']: 145 fail_str = 'No suitable backend:' 146 elif test_id in ['t008']: 147 fail_str = 'Available backend resources:' 148 elif test_id in ['t110', 't111', 't112', 't113', 't114']: 149 fail_str = 'Cannot grant CeedVector array access' 150 elif test_id in ['t115']: 151 fail_str = 'Cannot grant CeedVector read-only array access, the access lock is already in use' 152 elif test_id in ['t116']: 153 fail_str = 'Cannot destroy CeedVector, the writable access lock is in use' 154 elif test_id in ['t117']: 155 fail_str = 'Cannot restore CeedVector array access, access was not granted' 156 elif test_id in ['t118']: 157 fail_str = 'Cannot sync CeedVector, the access lock is already in use' 158 elif test_id in ['t215']: 159 fail_str = 'Cannot destroy CeedElemRestriction, a process has read access to the offset data' 160 elif test_id in ['t303']: 161 fail_str = 'Length of input/output vectors incompatible with basis dimensions' 162 elif test_id in ['t408']: 163 fail_str = 'CeedQFunctionContextGetData(): Cannot grant CeedQFunctionContext data access, a process has read access' 164 elif test_id in ['t409'] and contains_any(resource, ['memcheck']): 165 fail_str = 'Context data changed while accessed in read-only mode' 166 167 return fail_str, fail_str in stderr 168 169 def check_allowed_stdout(self, test: str) -> bool: 170 """Check whether a test is allowed to print console output 171 172 Args: 173 test (str): Name of test 174 175 Returns: 176 bool: True if the test is allowed to print console output 177 """ 178 return test[:4] in ['t003'] 179 180 181if __name__ == '__main__': 182 args = create_argparser().parse_args() 183 184 # run tests 185 if 'smartsim' in args.test: 186 has_smartsim: bool = args.smartredis_dir and Path(args.smartredis_dir).is_dir() 187 test_cases = [] 188 189 if args.mode is RunMode.TAP: 190 print(f'1..1') 191 if has_smartsim: 192 sys.path.insert(0, str(Path(__file__).parents[1] / "examples" / "fluids")) 193 from smartsim_regression_framework import SmartSimTest 194 195 test_framework = SmartSimTest(Path(__file__).parent / 'test_dir') 196 test_framework.setup() 197 198 is_new_subtest = True 199 subtest_ok = True 200 for i, backend in enumerate(args.ceed_backends): 201 test_cases.append(test_framework.test_junit(backend)) 202 if is_new_subtest and args.mode == RunMode.TAP: 203 is_new_subtest = False 204 print(f'# Subtest: {test_cases[0].category}') 205 print(f' 1..{len(args.ceed_backends)}') 206 print(test_case_output_string(test_cases[i], TestSpec("SmartSim Tests"), args.mode, backend, '', i)) 207 if args.mode == RunMode.TAP: 208 print(f'{"" if subtest_ok else "not "}ok 1 - {test_cases[0].category}') 209 test_framework.teardown() 210 elif args.mode is RunMode.TAP: 211 print(f'ok 1 - # SKIP SmartSim not installed') 212 result: TestSuite = TestSuite('SmartSim Tests', test_cases) 213 else: 214 result: TestSuite = run_tests( 215 args.test, 216 args.ceed_backends, 217 args.mode, 218 args.nproc, 219 CeedSuiteSpec(args.has_torch), 220 args.pool_size) 221 222 # write output and check for failures 223 if args.mode is RunMode.JUNIT: 224 write_junit_xml(result, args.output, args.junit_batch) 225 if has_failures(result): 226 sys.exit(1) 227