1from abc import ABC, abstractmethod 2from collections.abc import Iterable 3import argparse 4import csv 5from dataclasses import dataclass, field, fields 6import difflib 7from enum import Enum 8from math import isclose 9import os 10from pathlib import Path 11import re 12import subprocess 13import multiprocessing as mp 14import sys 15import time 16from typing import Optional, Tuple, List, Dict, Callable, Iterable, get_origin 17import shutil 18 19sys.path.insert(0, str(Path(__file__).parent / "junit-xml")) 20from junit_xml import TestCase, TestSuite, to_xml_report_string # nopep8 21 22 23class ParseError(RuntimeError): 24 """A custom exception for failed parsing.""" 25 26 def __init__(self, message): 27 super().__init__(message) 28 29 30class CaseInsensitiveEnumAction(argparse.Action): 31 """Action to convert input values to lower case prior to converting to an Enum type""" 32 33 def __init__(self, option_strings, dest, type, default, **kwargs): 34 if not issubclass(type, Enum): 35 raise ValueError(f"{type} must be an Enum") 36 # store provided enum type 37 self.enum_type = type 38 if isinstance(default, self.enum_type): 39 pass 40 elif isinstance(default, str): 41 default = self.enum_type(default.lower()) 42 elif isinstance(default, Iterable): 43 default = [self.enum_type(v.lower()) for v in default] 44 else: 45 raise argparse.ArgumentTypeError("Invalid value type, must be str or iterable") 46 # prevent automatic type conversion 47 super().__init__(option_strings, dest, default=default, **kwargs) 48 49 def __call__(self, parser, namespace, values, option_string=None): 50 if isinstance(values, self.enum_type): 51 pass 52 elif isinstance(values, str): 53 values = self.enum_type(values.lower()) 54 elif isinstance(values, Iterable): 55 values = [self.enum_type(v.lower()) for v in values] 56 else: 57 raise argparse.ArgumentTypeError("Invalid value type, must be str or iterable") 58 setattr(namespace, self.dest, values) 59 60 61@dataclass 62class TestSpec: 63 """Dataclass storing information about a single test case""" 64 name: str = field(default_factory=str) 65 csv_rtol: float = -1 66 csv_ztol: float = -1 67 cgns_tol: float = -1 68 only: List = field(default_factory=list) 69 args: List = field(default_factory=list) 70 key_values: Dict = field(default_factory=dict) 71 72 73class RunMode(Enum): 74 """Enumeration of run modes, either `RunMode.TAP` or `RunMode.JUNIT`""" 75 TAP = 'tap' 76 JUNIT = 'junit' 77 78 def __str__(self): 79 return self.value 80 81 def __repr__(self): 82 return self.value 83 84 85class SuiteSpec(ABC): 86 """Abstract Base Class defining the required interface for running a test suite""" 87 @abstractmethod 88 def get_source_path(self, test: str) -> Path: 89 """Compute path to test source file 90 91 Args: 92 test (str): Name of test 93 94 Returns: 95 Path: Path to source file 96 """ 97 raise NotImplementedError 98 99 @abstractmethod 100 def get_run_path(self, test: str) -> Path: 101 """Compute path to built test executable file 102 103 Args: 104 test (str): Name of test 105 106 Returns: 107 Path: Path to test executable 108 """ 109 raise NotImplementedError 110 111 @abstractmethod 112 def get_output_path(self, test: str, output_file: str) -> Path: 113 """Compute path to expected output file 114 115 Args: 116 test (str): Name of test 117 output_file (str): File name of output file 118 119 Returns: 120 Path: Path to expected output file 121 """ 122 raise NotImplementedError 123 124 @property 125 def test_failure_artifacts_path(self) -> Path: 126 """Path to test failure artifacts""" 127 return Path('build') / 'test_failure_artifacts' 128 129 @property 130 def cgns_tol(self): 131 """Absolute tolerance for CGNS diff""" 132 return getattr(self, '_cgns_tol', 1.0e-12) 133 134 @cgns_tol.setter 135 def cgns_tol(self, val): 136 self._cgns_tol = val 137 138 @property 139 def csv_ztol(self): 140 """Keyword arguments to be passed to diff_csv()""" 141 return getattr(self, '_csv_ztol', 3e-10) 142 143 @csv_ztol.setter 144 def csv_ztol(self, val): 145 self._csv_ztol = val 146 147 @property 148 def csv_rtol(self): 149 """Keyword arguments to be passed to diff_csv()""" 150 return getattr(self, '_csv_rtol', 1e-6) 151 152 @csv_rtol.setter 153 def csv_rtol(self, val): 154 self._csv_rtol = val 155 156 @property 157 def csv_comment_diff_fn(self): # -> Any | Callable[..., None]: 158 return getattr(self, '_csv_comment_diff_fn', None) 159 160 @csv_comment_diff_fn.setter 161 def csv_comment_diff_fn(self, test_fn): 162 self._csv_comment_diff_fn = test_fn 163 164 @property 165 def csv_comment_str(self): 166 return getattr(self, '_csv_comment_str', '#') 167 168 @csv_comment_str.setter 169 def csv_comment_str(self, comment_str): 170 self._csv_comment_str = comment_str 171 172 def post_test_hook(self, test: str, spec: TestSpec, backend: str) -> None: 173 """Function callback ran after each test case 174 175 Args: 176 test (str): Name of test 177 spec (TestSpec): Test case specification 178 """ 179 pass 180 181 def check_pre_skip(self, test: str, spec: TestSpec, resource: str, nproc: int) -> Optional[str]: 182 """Check if a test case should be skipped prior to running, returning the reason for skipping 183 184 Args: 185 test (str): Name of test 186 spec (TestSpec): Test case specification 187 resource (str): libCEED backend 188 nproc (int): Number of MPI processes to use when running test case 189 190 Returns: 191 Optional[str]: Skip reason, or `None` if test case should not be skipped 192 """ 193 return None 194 195 def check_post_skip(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Optional[str]: 196 """Check if a test case should be allowed to fail, based on its stderr output 197 198 Args: 199 test (str): Name of test 200 spec (TestSpec): Test case specification 201 resource (str): libCEED backend 202 stderr (str): Standard error output from test case execution 203 204 Returns: 205 Optional[str]: Skip reason, or `None` if unexpected error 206 """ 207 return None 208 209 def check_required_failure(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Tuple[str, bool]: 210 """Check whether a test case is expected to fail and if it failed expectedly 211 212 Args: 213 test (str): Name of test 214 spec (TestSpec): Test case specification 215 resource (str): libCEED backend 216 stderr (str): Standard error output from test case execution 217 218 Returns: 219 tuple[str, bool]: Tuple of the expected failure string and whether it was present in `stderr` 220 """ 221 return '', True 222 223 def check_allowed_stdout(self, test: str) -> bool: 224 """Check whether a test is allowed to print console output 225 226 Args: 227 test (str): Name of test 228 229 Returns: 230 bool: True if the test is allowed to print console output 231 """ 232 return False 233 234 235def has_cgnsdiff() -> bool: 236 """Check whether `cgnsdiff` is an executable program in the current environment 237 238 Returns: 239 bool: True if `cgnsdiff` is found 240 """ 241 my_env: dict = os.environ.copy() 242 proc = subprocess.run('cgnsdiff', 243 shell=True, 244 stdout=subprocess.PIPE, 245 stderr=subprocess.PIPE, 246 env=my_env) 247 return 'not found' not in proc.stderr.decode('utf-8') 248 249 250def contains_any(base: str, substrings: List[str]) -> bool: 251 """Helper function, checks if any of the substrings are included in the base string 252 253 Args: 254 base (str): Base string to search in 255 substrings (List[str]): List of potential substrings 256 257 Returns: 258 bool: True if any substrings are included in base string 259 """ 260 return any((sub in base for sub in substrings)) 261 262 263def startswith_any(base: str, prefixes: List[str]) -> bool: 264 """Helper function, checks if the base string is prefixed by any of `prefixes` 265 266 Args: 267 base (str): Base string to search 268 prefixes (List[str]): List of potential prefixes 269 270 Returns: 271 bool: True if base string is prefixed by any of the prefixes 272 """ 273 return any((base.startswith(prefix) for prefix in prefixes)) 274 275 276def find_matching(line: str, open: str = '(', close: str = ')') -> Tuple[int, int]: 277 """Find the start and end positions of the first outer paired delimeters 278 279 Args: 280 line (str): Line to search 281 open (str, optional): Opening delimiter, must be different than `close`. Defaults to '('. 282 close (str, optional): Closing delimeter, must be different than `open`. Defaults to ')'. 283 284 Raises: 285 RuntimeError: If open or close is not a single character 286 RuntimeError: If open and close are the same characters 287 288 Returns: 289 Tuple[int]: If matching delimeters are found, return indices in `list`. Otherwise, return end < start. 290 """ 291 if len(open) != 1 or len(close) != 1: 292 raise RuntimeError("`open` and `close` must be single characters") 293 if open == close: 294 raise RuntimeError("`open` and `close` must be different characters") 295 start: int = line.find(open) 296 if start < 0: 297 return -1, -1 298 count: int = 1 299 for i in range(start + 1, len(line)): 300 if line[i] == open: 301 count += 1 302 if line[i] == close: 303 count -= 1 304 if count == 0: 305 return start, i 306 return start, -1 307 308 309def parse_test_line(line: str, fallback_name: str = '') -> TestSpec: 310 """Parse a single line of TESTARGS and CLI arguments into a `TestSpec` object 311 312 Args: 313 line (str): String containing TESTARGS specification and CLI arguments 314 315 Returns: 316 TestSpec: Parsed specification of test case 317 """ 318 test_fields = fields(TestSpec) 319 field_names = [f.name for f in test_fields] 320 known: Dict = dict() 321 other: Dict = dict() 322 if line[0] == "(": 323 # have key/value pairs to parse 324 start, end = find_matching(line) 325 if end < start: 326 raise ParseError(f"Mismatched parentheses in TESTCASE: {line}") 327 328 keyvalues_str = line[start:end + 1] 329 keyvalues_pattern = re.compile(r''' 330 (?:\(\s*|\s*,\s*) # start with open parentheses or comma, no capture 331 ([A-Za-z]+[\w\-]+) # match key starting with alpha, containing alphanumeric, _, or -; captured as Group 1 332 \s*=\s* # key is followed by = (whitespace ignored) 333 (?: # uncaptured group for OR 334 "((?:[^"]|\\")+)" # match quoted value (any internal " must be escaped as \"); captured as Group 2 335 | ([^=]+) # OR match unquoted value (no equals signs allowed); captured as Group 3 336 ) # end uncaptured group for OR 337 \s*(?=,|\)) # lookahead for either next comma or closing parentheses 338 ''', re.VERBOSE) 339 340 for match in re.finditer(keyvalues_pattern, keyvalues_str): 341 if not match: # empty 342 continue 343 key = match.group(1) 344 value = match.group(2) if match.group(2) else match.group(3) 345 try: 346 index = field_names.index(key) 347 if key == "only": # weird bc only is a list 348 value = [constraint.strip() for constraint in value.split(',')] 349 try: 350 # TODO: stop supporting python <=3.8 351 known[key] = test_fields[index].type(value) # type: ignore 352 except TypeError: 353 # TODO: this is still liable to fail for complex types 354 known[key] = get_origin(test_fields[index].type)(value) # type: ignore 355 except ValueError: 356 other[key] = value 357 358 line = line[end + 1:] 359 360 if not 'name' in known.keys(): 361 known['name'] = fallback_name 362 363 args_pattern = re.compile(r''' 364 \s+( # remove leading space 365 (?:"[^"]+") # match quoted CLI option 366 | (?:[\S]+) # match anything else that is space separated 367 ) 368 ''', re.VERBOSE) 369 args: List[str] = re.findall(args_pattern, line) 370 for k, v in other.items(): 371 print(f"warning, unknown TESTCASE option for test '{known['name']}': {k}={v}") 372 return TestSpec(**known, key_values=other, args=args) 373 374 375def get_test_args(source_file: Path) -> List[TestSpec]: 376 """Parse all test cases from a given source file 377 378 Args: 379 source_file (Path): Path to source file 380 381 Raises: 382 RuntimeError: Errors if source file extension is unsupported 383 384 Returns: 385 List[TestSpec]: List of parsed `TestSpec` objects, or a list containing a single, default `TestSpec` if none were found 386 """ 387 comment_str: str = '' 388 if source_file.suffix in ['.c', '.cc', '.cpp']: 389 comment_str = '//' 390 elif source_file.suffix in ['.py']: 391 comment_str = '#' 392 elif source_file.suffix in ['.usr']: 393 comment_str = 'C_' 394 elif source_file.suffix in ['.f90']: 395 comment_str = '! ' 396 else: 397 raise RuntimeError(f'Unrecognized extension for file: {source_file}') 398 399 return [parse_test_line(line.strip(comment_str).removeprefix("TESTARGS"), source_file.stem) 400 for line in source_file.read_text().splitlines() 401 if line.startswith(f'{comment_str}TESTARGS')] or [TestSpec(source_file.stem, args=['{ceed_resource}'])] 402 403 404def diff_csv(test_csv: Path, true_csv: Path, zero_tol: float, rel_tol: float, 405 comment_str: str = '#', comment_func: Optional[Callable[[str, str], Optional[str]]] = None) -> str: 406 """Compare CSV results against an expected CSV file with tolerances 407 408 Args: 409 test_csv (Path): Path to output CSV results 410 true_csv (Path): Path to expected CSV results 411 zero_tol (float): Tolerance below which values are considered to be zero. 412 rel_tol (float): Relative tolerance for comparing non-zero values. 413 comment_str (str, optional): String to denoting commented line 414 comment_func (Callable, optional): Function to determine if test and true line are different 415 416 Returns: 417 str: Diff output between result and expected CSVs 418 """ 419 test_lines: List[str] = test_csv.read_text().splitlines() 420 true_lines: List[str] = true_csv.read_text().splitlines() 421 # Files should not be empty 422 if len(test_lines) == 0: 423 return f'No lines found in test output {test_csv}' 424 if len(true_lines) == 0: 425 return f'No lines found in test source {true_csv}' 426 if len(test_lines) != len(true_lines): 427 return f'Number of lines in {test_csv} and {true_csv} do not match' 428 429 # Process commented lines 430 uncommented_lines: List[int] = [] 431 for n, (test_line, true_line) in enumerate(zip(test_lines, true_lines)): 432 if test_line[0] == comment_str and true_line[0] == comment_str: 433 if comment_func: 434 output = comment_func(test_line, true_line) 435 if output: 436 return output 437 elif test_line[0] == comment_str and true_line[0] != comment_str: 438 return f'Commented line found in {test_csv} at line {n} but not in {true_csv}' 439 elif test_line[0] != comment_str and true_line[0] == comment_str: 440 return f'Commented line found in {true_csv} at line {n} but not in {test_csv}' 441 else: 442 uncommented_lines.append(n) 443 444 # Remove commented lines 445 test_lines = [test_lines[line] for line in uncommented_lines] 446 true_lines = [true_lines[line] for line in uncommented_lines] 447 448 test_reader: csv.DictReader = csv.DictReader(test_lines) 449 true_reader: csv.DictReader = csv.DictReader(true_lines) 450 if not test_reader.fieldnames: 451 return f'No CSV columns found in test output {test_csv}' 452 if not true_reader.fieldnames: 453 return f'No CSV columns found in test source {true_csv}' 454 if test_reader.fieldnames != true_reader.fieldnames: 455 return ''.join(difflib.unified_diff([f'{test_lines[0]}\n'], [f'{true_lines[0]}\n'], 456 tofile='found CSV columns', fromfile='expected CSV columns')) 457 458 diff_lines: List[str] = list() 459 for test_line, true_line in zip(test_reader, true_reader): 460 for key in test_reader.fieldnames: 461 # Check if the value is numeric 462 try: 463 true_val: float = float(true_line[key]) 464 test_val: float = float(test_line[key]) 465 true_zero: bool = abs(true_val) < zero_tol 466 test_zero: bool = abs(test_val) < zero_tol 467 fail: bool = False 468 if true_zero: 469 fail = not test_zero 470 else: 471 fail = not isclose(test_val, true_val, rel_tol=rel_tol) 472 if fail: 473 diff_lines.append(f'column: {key}, expected: {true_val}, got: {test_val}') 474 except ValueError: 475 if test_line[key] != true_line[key]: 476 diff_lines.append(f'column: {key}, expected: {true_line[key]}, got: {test_line[key]}') 477 478 return '\n'.join(diff_lines) 479 480 481def diff_cgns(test_cgns: Path, true_cgns: Path, cgns_tol: float) -> str: 482 """Compare CGNS results against an expected CGSN file with tolerance 483 484 Args: 485 test_cgns (Path): Path to output CGNS file 486 true_cgns (Path): Path to expected CGNS file 487 cgns_tol (float): Tolerance for comparing floating-point values 488 489 Returns: 490 str: Diff output between result and expected CGNS files 491 """ 492 my_env: dict = os.environ.copy() 493 494 run_args: List[str] = ['cgnsdiff', '-d', '-t', f'{cgns_tol}', str(test_cgns), str(true_cgns)] 495 proc = subprocess.run(' '.join(run_args), 496 shell=True, 497 stdout=subprocess.PIPE, 498 stderr=subprocess.PIPE, 499 env=my_env) 500 501 return proc.stderr.decode('utf-8') + proc.stdout.decode('utf-8') 502 503 504def diff_ascii(test_file: Path, true_file: Path, backend: str) -> str: 505 """Compare ASCII results against an expected ASCII file 506 507 Args: 508 test_file (Path): Path to output ASCII file 509 true_file (Path): Path to expected ASCII file 510 511 Returns: 512 str: Diff output between result and expected ASCII files 513 """ 514 tmp_backend: str = backend.replace('/', '-') 515 true_str: str = true_file.read_text().replace('{ceed_resource}', tmp_backend) 516 diff = list(difflib.unified_diff(test_file.read_text().splitlines(keepends=True), 517 true_str.splitlines(keepends=True), 518 fromfile=str(test_file), 519 tofile=str(true_file))) 520 return ''.join(diff) 521 522 523def test_case_output_string(test_case: TestCase, spec: TestSpec, mode: RunMode, 524 backend: str, test: str, index: int, verbose: bool) -> str: 525 output_str = '' 526 if mode is RunMode.TAP: 527 # print incremental output if TAP mode 528 if test_case.is_skipped(): 529 output_str += f' ok {index} - {spec.name}, {backend} # SKIP {test_case.skipped[0]["message"]}\n' 530 elif test_case.is_failure() or test_case.is_error(): 531 output_str += f' not ok {index} - {spec.name}, {backend} ({test_case.elapsed_sec} s)\n' 532 else: 533 output_str += f' ok {index} - {spec.name}, {backend} ({test_case.elapsed_sec} s)\n' 534 if test_case.is_failure() or test_case.is_error() or verbose: 535 output_str += f' ---\n' 536 if spec.only: 537 output_str += f' only: {",".join(spec.only)}\n' 538 output_str += f' args: {test_case.args}\n' 539 if spec.csv_ztol > 0: 540 output_str += f' csv_ztol: {spec.csv_ztol}\n' 541 if spec.csv_rtol > 0: 542 output_str += f' csv_rtol: {spec.csv_rtol}\n' 543 if spec.cgns_tol > 0: 544 output_str += f' cgns_tol: {spec.cgns_tol}\n' 545 for k, v in spec.key_values.items(): 546 output_str += f' {k}: {v}\n' 547 if test_case.is_error(): 548 output_str += f' error: {test_case.errors[0]["message"]}\n' 549 if test_case.is_failure(): 550 output_str += f' failures:\n' 551 for i, failure in enumerate(test_case.failures): 552 output_str += f' -\n' 553 output_str += f' message: {failure["message"]}\n' 554 if failure["output"]: 555 out = failure["output"].strip().replace('\n', '\n ') 556 output_str += f' output: |\n {out}\n' 557 output_str += f' ...\n' 558 else: 559 # print error or failure information if JUNIT mode 560 if test_case.is_error() or test_case.is_failure(): 561 output_str += f'Test: {test} {spec.name}\n' 562 output_str += f' $ {test_case.args}\n' 563 if test_case.is_error(): 564 output_str += 'ERROR: {}\n'.format((test_case.errors[0]['message'] or 'NO MESSAGE').strip()) 565 output_str += 'Output: \n{}\n'.format((test_case.errors[0]['output'] or 'NO MESSAGE').strip()) 566 if test_case.is_failure(): 567 for failure in test_case.failures: 568 output_str += 'FAIL: {}\n'.format((failure['message'] or 'NO MESSAGE').strip()) 569 output_str += 'Output: \n{}\n'.format((failure['output'] or 'NO MESSAGE').strip()) 570 return output_str 571 572 573def save_failure_artifact(suite_spec: SuiteSpec, file: Path) -> Path: 574 """Attach a file to a test case 575 576 Args: 577 test_case (TestCase): Test case to attach the file to 578 file (Path): Path to the file to attach 579 """ 580 save_path: Path = suite_spec.test_failure_artifacts_path / file.name 581 shutil.copyfile(file, save_path) 582 return save_path 583 584 585def run_test(index: int, test: str, spec: TestSpec, backend: str, 586 mode: RunMode, nproc: int, suite_spec: SuiteSpec, verbose: bool = False) -> TestCase: 587 """Run a single test case and backend combination 588 589 Args: 590 index (int): Index of backend for current spec 591 test (str): Path to test 592 spec (TestSpec): Specification of test case 593 backend (str): CEED backend 594 mode (RunMode): Output mode 595 nproc (int): Number of MPI processes to use when running test case 596 suite_spec (SuiteSpec): Specification of test suite 597 verbose (bool, optional): Print detailed output for all runs, not just failures. Defaults to False. 598 599 Returns: 600 TestCase: Test case result 601 """ 602 source_path: Path = suite_spec.get_source_path(test) 603 run_args: List = [f'{suite_spec.get_run_path(test)}', *map(str, spec.args)] 604 605 if '{ceed_resource}' in run_args: 606 run_args[run_args.index('{ceed_resource}')] = backend 607 for i, arg in enumerate(run_args): 608 if '{ceed_resource}' in arg: 609 run_args[i] = arg.replace('{ceed_resource}', backend.replace('/', '-')) 610 if '{nproc}' in run_args: 611 run_args[run_args.index('{nproc}')] = f'{nproc}' 612 elif nproc > 1 and source_path.suffix != '.py': 613 run_args = ['mpiexec', '-n', f'{nproc}', *run_args] 614 615 # run test 616 skip_reason: Optional[str] = suite_spec.check_pre_skip(test, spec, backend, nproc) 617 if skip_reason: 618 test_case: TestCase = TestCase(f'{test}, "{spec.name}", n{nproc}, {backend}', 619 elapsed_sec=0, 620 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()), 621 stdout='', 622 stderr='', 623 category=spec.name,) 624 test_case.add_skipped_info(skip_reason) 625 else: 626 start: float = time.time() 627 proc = subprocess.run(' '.join(str(arg) for arg in run_args), 628 shell=True, 629 stdout=subprocess.PIPE, 630 stderr=subprocess.PIPE, 631 env=my_env) 632 633 test_case = TestCase(f'{test}, "{spec.name}", n{nproc}, {backend}', 634 classname=source_path.parent, 635 elapsed_sec=time.time() - start, 636 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 637 stdout=proc.stdout.decode('utf-8'), 638 stderr=proc.stderr.decode('utf-8'), 639 allow_multiple_subelements=True, 640 category=spec.name,) 641 ref_csvs: List[Path] = [] 642 ref_ascii: List[Path] = [] 643 output_files: List[str] = [arg.split(':')[1] for arg in run_args if arg.startswith('ascii:')] 644 if output_files: 645 ref_csvs = [suite_spec.get_output_path(test, file) 646 for file in output_files if file.endswith('.csv')] 647 ref_ascii = [suite_spec.get_output_path(test, file) 648 for file in output_files if not file.endswith('.csv')] 649 ref_cgns: List[Path] = [] 650 output_files = [arg.split(':')[1] for arg in run_args if arg.startswith('cgns:')] 651 if output_files: 652 ref_cgns = [suite_spec.get_output_path(test, file) for file in output_files] 653 ref_stdout: Path = suite_spec.get_output_path(test, test + '.out') 654 suite_spec.post_test_hook(test, spec, backend) 655 656 # check allowed failures 657 if not test_case.is_skipped() and test_case.stderr: 658 skip_reason: Optional[str] = suite_spec.check_post_skip(test, spec, backend, test_case.stderr) 659 if skip_reason: 660 test_case.add_skipped_info(skip_reason) 661 662 # check required failures 663 if not test_case.is_skipped(): 664 required_message, did_fail = suite_spec.check_required_failure( 665 test, spec, backend, test_case.stderr) 666 if required_message and did_fail: 667 test_case.status = f'fails with required: {required_message}' 668 elif required_message: 669 test_case.add_failure_info(f'required failure missing: {required_message}') 670 671 # classify other results 672 if not test_case.is_skipped() and not test_case.status: 673 if test_case.stderr: 674 test_case.add_failure_info('stderr', test_case.stderr) 675 if proc.returncode != 0: 676 test_case.add_error_info(f'returncode = {proc.returncode}') 677 if ref_stdout.is_file(): 678 diff = list(difflib.unified_diff(ref_stdout.read_text().splitlines(keepends=True), 679 test_case.stdout.splitlines(keepends=True), 680 fromfile=str(ref_stdout), 681 tofile='New')) 682 if diff: 683 test_case.add_failure_info('stdout', output=''.join(diff)) 684 elif test_case.stdout and not suite_spec.check_allowed_stdout(test): 685 test_case.add_failure_info('stdout', output=test_case.stdout) 686 # expected CSV output 687 for ref_csv in ref_csvs: 688 csv_name = ref_csv.name 689 out_file = Path.cwd() / csv_name 690 if not ref_csv.is_file(): 691 # remove _{ceed_backend} from path name 692 ref_csv = (ref_csv.parent / ref_csv.name.rsplit('_', 1)[0]).with_suffix('.csv') 693 if not ref_csv.is_file(): 694 test_case.add_failure_info('csv', output=f'{ref_csv} not found') 695 elif not out_file.is_file(): 696 test_case.add_failure_info('csv', output=f'{out_file} not found') 697 else: 698 csv_ztol: float = spec.csv_ztol if spec.csv_ztol > 0 else suite_spec.csv_ztol 699 csv_rtol: float = spec.csv_rtol if spec.csv_rtol > 0 else suite_spec.csv_rtol 700 diff = diff_csv( 701 out_file, 702 ref_csv, 703 csv_ztol, 704 csv_rtol, 705 suite_spec.csv_comment_str, 706 suite_spec.csv_comment_diff_fn) 707 if diff: 708 save_path: Path = suite_spec.test_failure_artifacts_path / csv_name 709 shutil.move(out_file, save_path) 710 test_case.add_failure_info(f'csv: {save_path}', output=diff) 711 else: 712 out_file.unlink() 713 # expected CGNS output 714 for ref_cgn in ref_cgns: 715 cgn_name = ref_cgn.name 716 out_file = Path.cwd() / cgn_name 717 if not ref_cgn.is_file(): 718 # remove _{ceed_backend} from path name 719 ref_cgn = (ref_cgn.parent / ref_cgn.name.rsplit('_', 1)[0]).with_suffix('.cgns') 720 if not ref_cgn.is_file(): 721 test_case.add_failure_info('cgns', output=f'{ref_cgn} not found') 722 elif not out_file.is_file(): 723 test_case.add_failure_info('cgns', output=f'{out_file} not found') 724 else: 725 cgns_tol = spec.cgns_tol if spec.cgns_tol > 0 else suite_spec.cgns_tol 726 diff = diff_cgns(out_file, ref_cgn, cgns_tol=cgns_tol) 727 if diff: 728 save_path: Path = suite_spec.test_failure_artifacts_path / cgn_name 729 shutil.move(out_file, save_path) 730 test_case.add_failure_info(f'cgns: {save_path}', output=diff) 731 else: 732 out_file.unlink() 733 # expected ASCII output 734 for ref_file in ref_ascii: 735 ref_name = ref_file.name 736 out_file = Path.cwd() / ref_name 737 if not ref_file.is_file(): 738 # remove _{ceed_backend} from path name 739 ref_file = (ref_file.parent / ref_file.name.rsplit('_', 1)[0]).with_suffix(ref_file.suffix) 740 if not ref_file.is_file(): 741 test_case.add_failure_info('ascii', output=f'{ref_file} not found') 742 elif not out_file.is_file(): 743 test_case.add_failure_info('ascii', output=f'{out_file} not found') 744 else: 745 diff = diff_ascii(out_file, ref_file, backend) 746 if diff: 747 save_path: Path = suite_spec.test_failure_artifacts_path / ref_name 748 shutil.move(out_file, save_path) 749 test_case.add_failure_info(f'ascii: {save_path}', output=diff) 750 else: 751 out_file.unlink() 752 753 # store result 754 test_case.args = ' '.join(str(arg) for arg in run_args) 755 output_str = test_case_output_string(test_case, spec, mode, backend, test, index, verbose) 756 757 return test_case, output_str 758 759 760def init_process(): 761 """Initialize multiprocessing process""" 762 # set up error handler 763 global my_env 764 my_env = os.environ.copy() 765 my_env['CEED_ERROR_HANDLER'] = 'exit' 766 767 768def run_tests(test: str, ceed_backends: List[str], mode: RunMode, nproc: int, 769 suite_spec: SuiteSpec, pool_size: int = 1, search: str = ".*", verbose: bool = False) -> TestSuite: 770 """Run all test cases for `test` with each of the provided `ceed_backends` 771 772 Args: 773 test (str): Name of test 774 ceed_backends (List[str]): List of libCEED backends 775 mode (RunMode): Output mode, either `RunMode.TAP` or `RunMode.JUNIT` 776 nproc (int): Number of MPI processes to use when running each test case 777 suite_spec (SuiteSpec): Object defining required methods for running tests 778 pool_size (int, optional): Number of processes to use when running tests in parallel. Defaults to 1. 779 search (str, optional): Regular expression used to match tests. Defaults to ".*". 780 verbose (bool, optional): Print detailed output for all runs, not just failures. Defaults to False. 781 782 Returns: 783 TestSuite: JUnit `TestSuite` containing results of all test cases 784 """ 785 test_specs: List[TestSpec] = [ 786 t for t in get_test_args(suite_spec.get_source_path(test)) if re.search(search, t.name, re.IGNORECASE) 787 ] 788 suite_spec.test_failure_artifacts_path.mkdir(parents=True, exist_ok=True) 789 if mode is RunMode.TAP: 790 print('TAP version 13') 791 print(f'1..{len(test_specs)}') 792 793 with mp.Pool(processes=pool_size, initializer=init_process) as pool: 794 async_outputs: List[List[mp.pool.AsyncResult]] = [ 795 [pool.apply_async(run_test, (i, test, spec, backend, mode, nproc, suite_spec, verbose)) 796 for (i, backend) in enumerate(ceed_backends, start=1)] 797 for spec in test_specs 798 ] 799 800 test_cases = [] 801 for (i, subtest) in enumerate(async_outputs, start=1): 802 is_new_subtest = True 803 subtest_ok = True 804 for async_output in subtest: 805 test_case, print_output = async_output.get() 806 test_cases.append(test_case) 807 if is_new_subtest and mode == RunMode.TAP: 808 is_new_subtest = False 809 print(f'# Subtest: {test_case.category}') 810 print(f' 1..{len(ceed_backends)}') 811 print(print_output, end='') 812 if test_case.is_failure() or test_case.is_error(): 813 subtest_ok = False 814 if mode == RunMode.TAP: 815 print(f'{"" if subtest_ok else "not "}ok {i} - {test_case.category}') 816 817 return TestSuite(test, test_cases) 818 819 820def write_junit_xml(test_suite: TestSuite, output_file: Optional[Path], batch: str = '') -> None: 821 """Write a JUnit XML file containing the results of a `TestSuite` 822 823 Args: 824 test_suite (TestSuite): JUnit `TestSuite` to write 825 output_file (Optional[Path]): Path to output file, or `None` to generate automatically as `build/{test_suite.name}{batch}.junit` 826 batch (str): Name of JUnit batch, defaults to empty string 827 """ 828 output_file = output_file or Path('build') / (f'{test_suite.name}{batch}.junit') 829 output_file.write_text(to_xml_report_string([test_suite])) 830 831 832def has_failures(test_suite: TestSuite) -> bool: 833 """Check whether any test cases in a `TestSuite` failed 834 835 Args: 836 test_suite (TestSuite): JUnit `TestSuite` to check 837 838 Returns: 839 bool: True if any test cases failed 840 """ 841 return any(c.is_failure() or c.is_error() for c in test_suite.test_cases) 842