1#!/usr/bin/env python3 2""" 3# Created: Mon Jun 20 14:36:59 2022 (-0400) 4# @author: Jacob Faibussowitsch 5""" 6from __future__ import annotations 7 8import os 9import re 10import sys 11import traceback 12import subprocess 13import ctypes.util 14import clang.cindex as clx # type: ignore[import] 15 16from .._typing import * 17 18from ..__version__ import py_version_lt 19 20from ._clang import base_pch_clang_options 21 22def traceback_format_exception(exc: ExceptionKind) -> list[str]: 23 r"""Format an exception for printing 24 25 Parameters 26 ---------- 27 exc : 28 the exception instance 29 30 Returns 31 ------- 32 ret : 33 a list of lines which would have been printed had the exception been re-raised 34 """ 35 if py_version_lt(3, 10): 36 etype, value, tb = sys.exc_info() 37 ret = traceback.format_exception(etype, value, tb, chain=True) 38 else: 39 # type checkers do not grok that py_version_lt() means sys.version_info < (3, 10, 0) 40 ret = traceback.format_exception(exc, chain=True) # type: ignore [call-arg, arg-type] 41 return ret 42 43_T = TypeVar('_T') 44 45def subprocess_check_returncode(ret: subprocess.CompletedProcess[_T]) -> subprocess.CompletedProcess[_T]: 46 r"""Check the return code of a subprocess return value 47 48 Paramters 49 --------- 50 ret : 51 the return value of `subprocess.run()` 52 53 Returns 54 ------- 55 ret : 56 `ret` unchanged 57 58 Raises 59 ------ 60 RuntimeError 61 if `ret.returncode` is nonzero 62 """ 63 try: 64 ret.check_returncode() 65 except subprocess.CalledProcessError as cpe: 66 emess = '\n'.join([ 67 'Subprocess error:', 68 'stderr:', 69 f'{cpe.stderr}', 70 'stdout:', 71 f'{cpe.stdout}', 72 f'{cpe}' 73 ]) 74 raise RuntimeError(emess) from cpe 75 return ret 76 77def subprocess_capture_output(*args, **kwargs) -> subprocess.CompletedProcess[str]: 78 r"""Lightweight wrapper over subprocess.run 79 80 turns a subprocess.CalledProcessError into a RuntimeError with more diagnostics 81 82 Parameters 83 ---------- 84 *args : 85 arguments to subprocess.run 86 **kwargs : 87 keyword arguments to subprocess.run 88 89 Returns 90 ------- 91 ret : 92 the return value of `subprocess.run()` 93 94 Raises 95 ------ 96 RuntimeError 97 if `subprocess.run()` raises a `subprocess.CalledProcessError`, this routine converts it into a 98 RuntimeError with the output attached 99 """ 100 old_check = kwargs.get('check', True) 101 kwargs['check'] = False 102 ret = subprocess.run(*args, capture_output=True, universal_newlines=True, **kwargs) 103 if old_check: 104 ret = subprocess_check_returncode(ret) 105 return ret 106 107 108def initialize_libclang(clang_dir: Optional[StrPathLike] = None, clang_lib: Optional[StrPathLike] = None, compat_check: bool = True) -> tuple[Optional[StrPathLike], Optional[StrPathLike]]: 109 r"""Initialize libclang 110 111 Sets the required library file or directory path to initialize libclang 112 113 Parameters 114 ---------- 115 clang_dir : optional 116 the directory containing libclang 117 clang_lib : optional 118 the direct path to libclang 119 compat_check : optional 120 perform compatibility checks on loading the dynamic library 121 122 Returns 123 ------- 124 clang_dir, clang_lib : path_like 125 the resolved paths if loading occurred, otherwise the arguments unchanged 126 127 Raises 128 ------ 129 ValueError 130 if both `clang_dir` and `clang_lib` are None 131 132 Notes 133 ----- 134 If both `clang_lib` and `clang_dir` are given, `clang_lib` takes precedence. `clang_dir` is not 135 used in this instance. 136 """ 137 clxconf = clx.conf 138 if not clxconf.loaded: 139 from ..classes._path import Path 140 141 clxconf.set_compatibility_check(compat_check) 142 if clang_lib: 143 clang_lib = Path(clang_lib).resolve() 144 clxconf.set_library_file(str(clang_lib)) 145 elif clang_dir: 146 clang_dir = Path(clang_dir).resolve() 147 clxconf.set_library_path(str(clang_dir)) 148 else: 149 raise ValueError('Must supply either clang directory path or clang library path') 150 return clang_dir, clang_lib 151 152def try_to_find_libclang_dir() -> Optional[Path]: 153 r"""Crudely tries to find libclang directory. 154 155 First using ctypes.util.find_library(), then llvm-config, and then finally checks a few places on 156 macos 157 158 Returns 159 ------- 160 llvm_lib_dir : path_like | None 161 the path to libclang (i.e. LLVM_DIR/lib) or None if it was not found 162 """ 163 from ..classes._path import Path 164 165 llvm_lib_dir = ctypes.util.find_library('clang') 166 if not llvm_lib_dir: 167 try: 168 llvm_lib_dir = subprocess_capture_output(['llvm-config', '--libdir']).stdout.strip() 169 except FileNotFoundError: 170 # FileNotFoundError: [Errno 2] No such file or directory: 'llvm-config' 171 # try to find llvm_lib_dir by hand 172 import platform 173 174 if platform.system().casefold() == 'darwin': 175 try: 176 xcode_dir = subprocess_capture_output(['xcode-select', '-p']).stdout.strip() 177 if xcode_dir == '/Applications/Xcode.app/Contents/Developer': # default Xcode path 178 llvm_lib_dir = os.path.join( 179 xcode_dir, 'Toolchains', 'XcodeDefault.xctoolchain', 'usr', 'lib' 180 ) 181 elif xcode_dir == '/Library/Developer/CommandLineTools': # CLT path 182 llvm_lib_dir = os.path.join(xcode_dir, 'usr', 'lib') 183 except FileNotFoundError: 184 # FileNotFoundError: [Errno 2] No such file or directory: 'xcode-select' 185 pass 186 if not llvm_lib_dir: 187 return None 188 return Path(llvm_lib_dir).resolve(strict=True) 189 190def get_petsc_extra_includes(petsc_dir: Path, petsc_arch: str) -> list[str]: 191 r"""Retrieve the set of compiler flags to include PETSc libs 192 193 Parameters 194 ---------- 195 petsc_dir : path_like 196 the value of PETSC_DIR 197 petsc_arch : str 198 the value of PETSC_ARCH 199 200 Returns 201 ------- 202 ret : list 203 a list containing the flags to add to the compiler flags to pick up PETSc headers and configuration 204 """ 205 # keep these separate, since ORDER MATTERS HERE. Imagine that for example the 206 # mpiInclude dir has copies of old PETSc headers, you don't want these to come first 207 # in the include search path and hence override those found in petsc/include. 208 209 # You might be thinking that seems suspiciously specific, but I was this close to filing 210 # a bug report for python believing that cdll.load() was not deterministic... 211 petsc_includes = [] 212 mpi_includes = [] 213 raw_cxx_flags = [] 214 with open(petsc_dir/petsc_arch/'lib'/'petsc'/'conf'/'petscvariables', 'r') as pv: 215 cc_includes_re = re.compile(r'^PETSC_CC_INCLUDES\s*=') 216 mpi_includes_re = re.compile(r'^MPI_INCLUDE\s*=') 217 mpi_show_re = re.compile(r'^MPICC_SHOW\s*=') 218 cxx_flags_re = re.compile(r'^CXX_FLAGS\s*=') 219 220 def split_and_strip(line: str) -> list[str]: 221 return line.split('=', maxsplit=1)[1].split() 222 223 for line in pv: 224 if cc_includes_re.search(line): 225 petsc_includes.extend(split_and_strip(line)) 226 elif mpi_includes_re.search(line) or mpi_show_re.search(line): 227 mpi_includes.extend(split_and_strip(line)) 228 elif cxx_flags_re.search(line): 229 raw_cxx_flags.extend(split_and_strip(line)) 230 231 def filter_flags(flags: list[str], keep_prefix: str) -> Iterable[str]: 232 return (flag for flag in flags if flag.startswith(keep_prefix)) 233 234 std_flags = list(filter_flags(raw_cxx_flags, '-std=')) 235 cxx_flags = [std_flags[-1]] if std_flags else [] # take only the last one 236 237 include_gen = filter_flags(petsc_includes + mpi_includes, '-I') 238 seen: set[str] = set() 239 seen_add = seen.add 240 extra_includes = [flag for flag in include_gen if not flag in seen and not seen_add(flag)] 241 242 return cxx_flags + extra_includes 243 244def get_clang_sys_includes() -> list[str]: 245 r"""Get system clangs set of default include search directories. 246 247 Because for some reason these are hardcoded by the compilers and so libclang does not have them. 248 249 Returns 250 ------- 251 ret : 252 list of paths to append to compiler flags to pick up sys inclusions (e.g. <ctypes> or <stdlib.h>) 253 """ 254 from ..classes._path import Path 255 256 output = subprocess_capture_output(['clang', '-E', '-x', 'c++', os.devnull, '-v']) 257 # goes to stderr because of /dev/null 258 includes = output.stderr.split('#include <...> search starts here:\n')[1] 259 includes = includes.split('End of search list.', maxsplit=1)[0].replace('(framework directory)', '') 260 return [f'-I{Path(i.strip()).resolve()}' for i in includes.splitlines() if i] 261 262def build_compiler_flags(petsc_dir: Path, petsc_arch: str, extra_compiler_flags: Optional[list[str]] = None, verbose: int = 0) -> list[str]: 263 r"""Build the baseline set of compiler flags. 264 265 These are passed to all translation unit parse attempts. 266 267 Parameters 268 ---------- 269 petsc_dir : path_like | str 270 the value of PETSC_DIR 271 petsc_arch : str 272 the value of PETSC_ARCH 273 extra_compiler_flags : list[str] | None, optional 274 extra compiler flags, if None, an empty list is used 275 verbose : False, optional 276 print verbose output (at level) 277 278 Returns 279 ------- 280 compiler_flags : list[str] 281 the full list of compiler flags to pass to the parsers 282 """ 283 if extra_compiler_flags is None: 284 extra_compiler_flags = [] 285 286 misc_flags = [ 287 '-DPETSC_CLANG_STATIC_ANALYZER', 288 '-xc++', 289 '-Wno-empty-body', 290 '-Wno-writable-strings', 291 '-Wno-array-bounds', 292 '-Wno-nullability-completeness', 293 '-fparse-all-comments', 294 '-g' 295 ] 296 petsc_includes = get_petsc_extra_includes(petsc_dir, petsc_arch) 297 compiler_flags = get_clang_sys_includes() + misc_flags + petsc_includes + extra_compiler_flags 298 if verbose > 1: 299 import petsclinter as pl 300 301 pl.sync_print('\n'.join(['Compile flags:', *compiler_flags])) 302 return compiler_flags 303 304class PrecompiledHeader: 305 __slots__ = 'pch', 'verbose' 306 307 pch: PathLike 308 verbose: int 309 310 def __init__(self, pch: PathLike, verbose: int) -> None: 311 r"""Construct the PrecompiledHeader 312 313 Parameters 314 ---------- 315 pch : 316 the path where the precompiled header should be stored 317 verbose : 318 print verbose information (at level) 319 """ 320 self.pch = pch 321 self.verbose = verbose 322 return 323 324 def __enter__(self) -> PrecompiledHeader: 325 return self 326 327 def __exit__(self, *args, **kwargs) -> None: 328 if self.verbose: 329 import petsclinter as pl 330 331 pl.sync_print('Deleting precompiled header', self.pch) 332 self.pch.unlink() 333 return 334 335 @classmethod 336 def from_flags(cls, petsc_dir: Path, compiler_flags: list[str], extra_header_includes: Optional[list[str]] = None, verbose: int = 0, pch_clang_options: Optional[CXTranslationUnit] = None) -> PrecompiledHeader: 337 r"""Create a precompiled header from flags. 338 339 This builds the precompiled head from petsc.h, and all of the private headers. This not only saves 340 a lot of time, but is critical to finding struct definitions. Header contents are not parsed 341 during the actual linting, since this balloons the parsing time as libclang provides no builtin 342 auto header-precompilation like the normal compiler does. 343 344 Including petsc.h first should define almost everything we need so no side effects from including 345 headers in the wrong order below. 346 347 Parameters 348 ---------- 349 petsc_dir : path_like | str 350 the value of PETSC_DIR 351 compiler_flags : list[str] 352 the list of compiler flags to parse with 353 extra_header_includes : list[str], optional 354 extra header include directives to add 355 verbose : False, optional 356 print verbose information 357 pch_clang_options : iterable(int) 358 clang parsing options to use, if not set, petsclinter.util.base_pch_clang_options are used 359 360 Returns 361 ------- 362 ret : PrecompiledHeader 363 the precompiled header object 364 365 Raises 366 ------ 367 clang.cindex.LibclangError 368 if `extra_header_includes` is not None, and the compilation results in compiler diagnostics 369 """ 370 import petsclinter as pl 371 372 def verbose_print(*args, **kwargs) -> None: 373 if verbose > 1: 374 pl.sync_print(*args, **kwargs) 375 return 376 377 assert isinstance(petsc_dir, pl.Path) 378 if pch_clang_options is None: 379 pch_clang_options = base_pch_clang_options 380 381 if extra_header_includes is None: 382 extra_header_includes = [] 383 384 index = clx.Index.create() 385 precompiled_header = petsc_dir/'include'/'petsc_ast_precompile.pch' 386 mega_header_lines = [ 387 # Kokkos needs to go first since it mucks with complex 388 ('petscvec_kokkos.hpp', '#include <petscvec_kokkos.hpp>'), 389 ('petsc.h', '#include <petsc.h>') 390 ] 391 private_dir_name = petsc_dir/'include'/'petsc'/'private' 392 mega_header_name = 'mega_header.hpp' 393 394 # build a megaheader from every header in private first 395 for header in private_dir_name.iterdir(): 396 if header.suffix in ('.h', '.hpp'): 397 header_name = header.name 398 mega_header_lines.append((header_name, f'#include <petsc/private/{header_name}>')) 399 400 # loop until we get a completely clean compilation, any problematic headers are discarded 401 while True: 402 mega_header = '\n'.join(hfi for _,hfi in mega_header_lines)+'\n' # extra newline for last line 403 tu = index.parse( 404 mega_header_name, 405 args=compiler_flags, unsaved_files=[(mega_header_name, mega_header)], options=pch_clang_options 406 ) 407 diags = {} 408 for diag in tu.diagnostics: 409 try: 410 filename = diag.location.file.name 411 except AttributeError: 412 # file is None 413 continue 414 basename, filename = os.path.split(filename) 415 if filename not in diags: 416 # save the problematic header name as well as its path (a surprise tool that will 417 # help us later) 418 diags[filename] = (basename,diag) 419 for dirname, diag in tuple(diags.values()): 420 # the reason this is done twice is because as usual libclang hides 421 # everything in children. Suppose you have a primary header A (which might be 422 # include/petsc/private/headerA.h), header B and header C. Header B and C are in 423 # unknown locations and all we know is that Header A includes B which includes C. 424 # 425 # Now suppose header C is missing, meaning that Header A needs to be removed. 426 # libclang isn't gonna tell you that without some elbow grease since that would be 427 # far too easy. Instead it raises the error about header B, so we need to link it 428 # back to header A. 429 if dirname != private_dir_name: 430 # problematic header is NOT in include/petsc/private, so we have a header B on our 431 # hands 432 for child in diag.children: 433 # child of header B here is header A not header C 434 try: 435 filename = child.location.file.name 436 except AttributeError: 437 # file is None 438 continue 439 # filter out our fake header 440 if filename != mega_header_name: 441 # this will be include/petsc/private, headerA.h 442 basename, filename = os.path.split(filename) 443 if filename not in diags: 444 diags[filename] = (basename, diag) 445 if diags: 446 diagerrs = '\n'+'\n'.join(str(d) for _, d in diags.values()) 447 verbose_print('Included header has errors, removing', diagerrs) 448 mega_header_lines = [(hdr, hfi) for hdr, hfi in mega_header_lines if hdr not in diags] 449 else: 450 break 451 if extra_header_includes: 452 # now include the other headers but this time immediately crash on errors, let the 453 # user figure out their own busted header files 454 mega_header += '\n'.join(extra_header_includes) 455 verbose_print(f'Mega header:\n{mega_header}') 456 tu = index.parse( 457 mega_header_name, 458 args=compiler_flags, unsaved_files=[(mega_header_name, mega_header)], options=pch_clang_options 459 ) 460 if tu.diagnostics: 461 pl.sync_print('\n'.join(map(str, tu.diagnostics))) 462 raise clx.LibclangError('\n\nWarnings or errors generated when creating the precompiled header. This usually means that the provided libclang setup is faulty. If you used the auto-detection mechanism to find libclang then perhaps try specifying the location directly.') 463 else: 464 verbose_print(f'Mega header:\n{mega_header}') 465 precompiled_header.unlink(missing_ok=True) 466 tu.save(precompiled_header) 467 compiler_flags.extend(['-include-pch', str(precompiled_header)]) 468 verbose_print('Saving precompiled header', precompiled_header) 469 return cls(precompiled_header, verbose) 470