Lines Matching full:test
63 """Dataclass storing information about a single test case"""
86 """Abstract Base Class defining the required interface for running a test suite"""
88 def get_source_path(self, test: str) -> Path:
89 """Compute path to test source file
92 test (str): Name of test
100 def get_run_path(self, test: str) -> Path:
101 """Compute path to built test executable file
104 test (str): Name of test
107 Path: Path to test executable
112 def get_output_path(self, test: str, output_file: str) -> Path:
116 test (str): Name of test
126 """Path to test failure artifacts"""
172 def post_test_hook(self, test: str, spec: TestSpec, backend: str) -> None:
173 """Function callback ran after each test case
176 test (str): Name of test
177 spec (TestSpec): Test case specification
181 def check_pre_skip(self, test: str, spec: TestSpec, resource: str, nproc: int) -> Optional[str]:
182 … """Check if a test case should be skipped prior to running, returning the reason for skipping
185 test (str): Name of test
186 spec (TestSpec): Test case specification
188 nproc (int): Number of MPI processes to use when running test case
191 Optional[str]: Skip reason, or `None` if test case should not be skipped
195 … def check_post_skip(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Optional[str]:
196 """Check if a test case should be allowed to fail, based on its stderr output
199 test (str): Name of test
200 spec (TestSpec): Test case specification
202 stderr (str): Standard error output from test case execution
209 …def check_required_failure(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Tuple[s…
210 """Check whether a test case is expected to fail and if it failed expectedly
213 test (str): Name of test
214 spec (TestSpec): Test case specification
216 stderr (str): Standard error output from test case execution
223 def check_allowed_stdout(self, test: str) -> bool:
224 """Check whether a test is allowed to print console output
227 test (str): Name of test
230 bool: True if the test is allowed to print console output
316 TestSpec: Parsed specification of test case
371 print(f"warning, unknown TESTCASE option for test '{known['name']}': {k}={v}")
376 """Parse all test cases from a given source file
414 comment_func (Callable, optional): Function to determine if test and true line are different
423 return f'No lines found in test output {test_csv}'
425 return f'No lines found in test source {true_csv}'
451 return f'No CSV columns found in test output {test_csv}'
453 return f'No CSV columns found in test source {true_csv}'
524 backend: str, test: str, index: int, verbose: bool) -> str:
561 output_str += f'Test: {test} {spec.name}\n'
574 """Attach a file to a test case
577 test_case (TestCase): Test case to attach the file to
585 def run_test(index: int, test: str, spec: TestSpec, backend: str,
587 """Run a single test case and backend combination
591 test (str): Path to test
592 spec (TestSpec): Specification of test case
595 nproc (int): Number of MPI processes to use when running test case
596 suite_spec (SuiteSpec): Specification of test suite
600 TestCase: Test case result
602 source_path: Path = suite_spec.get_source_path(test)
603 run_args: List = [f'{suite_spec.get_run_path(test)}', *map(str, spec.args)]
615 # run test
616 skip_reason: Optional[str] = suite_spec.check_pre_skip(test, spec, backend, nproc)
618 test_case: TestCase = TestCase(f'{test}, "{spec.name}", n{nproc}, {backend}',
633 test_case = TestCase(f'{test}, "{spec.name}", n{nproc}, {backend}',
645 ref_csvs = [suite_spec.get_output_path(test, file)
647 ref_ascii = [suite_spec.get_output_path(test, file)
652 ref_cgns = [suite_spec.get_output_path(test, file) for file in output_files]
653 ref_stdout: Path = suite_spec.get_output_path(test, test + '.out')
654 suite_spec.post_test_hook(test, spec, backend)
658 … skip_reason: Optional[str] = suite_spec.check_post_skip(test, spec, backend, test_case.stderr)
665 test, spec, backend, test_case.stderr)
684 elif test_case.stdout and not suite_spec.check_allowed_stdout(test):
755 output_str = test_case_output_string(test_case, spec, mode, backend, test, index, verbose)
768 def run_tests(test: str, ceed_backends: List[str], mode: RunMode, nproc: int,
770 """Run all test cases for `test` with each of the provided `ceed_backends`
773 test (str): Name of test
776 nproc (int): Number of MPI processes to use when running each test case
783 TestSuite: JUnit `TestSuite` containing results of all test cases
786 …t for t in get_test_args(suite_spec.get_source_path(test)) if re.search(search, t.name, re.IGNOREC…
795 [pool.apply_async(run_test, (i, test, spec, backend, mode, nproc, suite_spec, verbose))
817 return TestSuite(test, test_cases)
832 """Check whether any test cases in a `TestSuite` failed
838 bool: True if any test cases failed