Lines Matching +full:test +full:-
19 sys.path.insert(0, str(Path(__file__).parent / "junit-xml"))
63 """Dataclass storing information about a single test case"""
65 csv_rtol: float = -1
66 csv_ztol: float = -1
67 cgns_tol: float = -1
86 """Abstract Base Class defining the required interface for running a test suite"""
88 def get_source_path(self, test: str) -> Path:
89 """Compute path to test source file
92 test (str): Name of test
100 def get_run_path(self, test: str) -> Path:
101 """Compute path to built test executable file
104 test (str): Name of test
107 Path: Path to test executable
112 def get_output_path(self, test: str, output_file: str) -> Path:
116 test (str): Name of test
125 def test_failure_artifacts_path(self) -> Path:
126 """Path to test failure artifacts"""
132 return getattr(self, '_cgns_tol', 1.0e-12)
141 return getattr(self, '_csv_ztol', 3e-10)
150 return getattr(self, '_csv_rtol', 1e-6)
157 def csv_comment_diff_fn(self): # -> Any | Callable[..., None]:
172 def post_test_hook(self, test: str, spec: TestSpec, backend: str) -> None:
173 """Function callback ran after each test case
176 test (str): Name of test
177 spec (TestSpec): Test case specification
181 def check_pre_skip(self, test: str, spec: TestSpec, resource: str, nproc: int) -> Optional[str]:
182 … """Check if a test case should be skipped prior to running, returning the reason for skipping
185 test (str): Name of test
186 spec (TestSpec): Test case specification
188 nproc (int): Number of MPI processes to use when running test case
191 Optional[str]: Skip reason, or `None` if test case should not be skipped
195 … def check_post_skip(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Optional[str]:
196 """Check if a test case should be allowed to fail, based on its stderr output
199 test (str): Name of test
200 spec (TestSpec): Test case specification
202 stderr (str): Standard error output from test case execution
209 …def check_required_failure(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Tuple[s…
210 """Check whether a test case is expected to fail and if it failed expectedly
213 test (str): Name of test
214 spec (TestSpec): Test case specification
216 stderr (str): Standard error output from test case execution
223 def check_allowed_stdout(self, test: str) -> bool:
224 """Check whether a test is allowed to print console output
227 test (str): Name of test
230 bool: True if the test is allowed to print console output
235 def has_cgnsdiff() -> bool:
247 return 'not found' not in proc.stderr.decode('utf-8')
250 def contains_any(base: str, substrings: List[str]) -> bool:
263 def startswith_any(base: str, prefixes: List[str]) -> bool:
276 def find_matching(line: str, open: str = '(', close: str = ')') -> Tuple[int, int]:
297 return -1, -1
303 count -= 1
306 return start, -1
309 def parse_test_line(line: str, fallback_name: str = '') -> TestSpec:
316 TestSpec: Parsed specification of test case
331 …([A-Za-z]+[\w\-]+) # match key starting with alpha, containing alphanumeric, _, or -; captured as…
371 print(f"warning, unknown TESTCASE option for test '{known['name']}': {k}={v}")
375 def get_test_args(source_file: Path) -> List[TestSpec]:
376 """Parse all test cases from a given source file
405 …comment_str: str = '#', comment_func: Optional[Callable[[str, str], Optional[str]]] = None) -> str:
412 rel_tol (float): Relative tolerance for comparing non-zero values.
414 comment_func (Callable, optional): Function to determine if test and true line are different
423 return f'No lines found in test output {test_csv}'
425 return f'No lines found in test source {true_csv}'
451 return f'No CSV columns found in test output {test_csv}'
453 return f'No CSV columns found in test source {true_csv}'
481 def diff_cgns(test_cgns: Path, true_cgns: Path, cgns_tol: float) -> str:
487 cgns_tol (float): Tolerance for comparing floating-point values
494 run_args: List[str] = ['cgnsdiff', '-d', '-t', f'{cgns_tol}', str(test_cgns), str(true_cgns)]
501 return proc.stderr.decode('utf-8') + proc.stdout.decode('utf-8')
504 def diff_ascii(test_file: Path, true_file: Path, backend: str) -> str:
514 tmp_backend: str = backend.replace('/', '-')
524 backend: str, test: str, index: int, verbose: bool) -> str:
529 …output_str += f' ok {index} - {spec.name}, {backend} # SKIP {test_case.skipped[0]["message"]}\n'
531 … output_str += f' not ok {index} - {spec.name}, {backend} ({test_case.elapsed_sec} s)\n'
533 output_str += f' ok {index} - {spec.name}, {backend} ({test_case.elapsed_sec} s)\n'
535 output_str += f' ---\n'
552 output_str += f' -\n'
561 output_str += f'Test: {test} {spec.name}\n'
573 def save_failure_artifact(suite_spec: SuiteSpec, file: Path) -> Path:
574 """Attach a file to a test case
577 test_case (TestCase): Test case to attach the file to
585 def run_test(index: int, test: str, spec: TestSpec, backend: str,
586 mode: RunMode, nproc: int, suite_spec: SuiteSpec, verbose: bool = False) -> TestCase:
587 """Run a single test case and backend combination
591 test (str): Path to test
592 spec (TestSpec): Specification of test case
595 nproc (int): Number of MPI processes to use when running test case
596 suite_spec (SuiteSpec): Specification of test suite
600 TestCase: Test case result
602 source_path: Path = suite_spec.get_source_path(test)
603 run_args: List = [f'{suite_spec.get_run_path(test)}', *map(str, spec.args)]
609 run_args[i] = arg.replace('{ceed_resource}', backend.replace('/', '-'))
613 run_args = ['mpiexec', '-n', f'{nproc}', *run_args]
615 # run test
616 skip_reason: Optional[str] = suite_spec.check_pre_skip(test, spec, backend, nproc)
618 test_case: TestCase = TestCase(f'{test}, "{spec.name}", n{nproc}, {backend}',
620 … timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()),
633 test_case = TestCase(f'{test}, "{spec.name}", n{nproc}, {backend}',
635 elapsed_sec=time.time() - start,
636 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)),
637 stdout=proc.stdout.decode('utf-8'),
638 stderr=proc.stderr.decode('utf-8'),
645 ref_csvs = [suite_spec.get_output_path(test, file)
647 ref_ascii = [suite_spec.get_output_path(test, file)
652 ref_cgns = [suite_spec.get_output_path(test, file) for file in output_files]
653 ref_stdout: Path = suite_spec.get_output_path(test, test + '.out')
654 suite_spec.post_test_hook(test, spec, backend)
658 … skip_reason: Optional[str] = suite_spec.check_post_skip(test, spec, backend, test_case.stderr)
665 test, spec, backend, test_case.stderr)
684 elif test_case.stdout and not suite_spec.check_allowed_stdout(test):
755 output_str = test_case_output_string(test_case, spec, mode, backend, test, index, verbose)
768 def run_tests(test: str, ceed_backends: List[str], mode: RunMode, nproc: int,
769 …suite_spec: SuiteSpec, pool_size: int = 1, search: str = ".*", verbose: bool = False) -> TestSuite:
770 """Run all test cases for `test` with each of the provided `ceed_backends`
773 test (str): Name of test
776 nproc (int): Number of MPI processes to use when running each test case
783 TestSuite: JUnit `TestSuite` containing results of all test cases
786 …t for t in get_test_args(suite_spec.get_source_path(test)) if re.search(search, t.name, re.IGNOREC…
795 [pool.apply_async(run_test, (i, test, spec, backend, mode, nproc, suite_spec, verbose))
815 print(f'{"" if subtest_ok else "not "}ok {i} - {test_case.category}')
817 return TestSuite(test, test_cases)
820 def write_junit_xml(test_suite: TestSuite, batch: str = '') -> None:
831 def has_failures(test_suite: TestSuite) -> bool:
832 """Check whether any test cases in a `TestSuite` failed
838 bool: True if any test cases failed