1#!/usr/bin/env python3 2 3import os 4from sysconfig import _parse_makefile as parse_makefile 5import sys 6import logging 7sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 8from collections import defaultdict 9 10AUTODIRS = set('ftn-auto ftn-custom f90-custom ftn-auto-interfaces'.split()) # Automatically recurse into these, if they exist 11SKIPDIRS = set('benchmarks build mex-scripts tests tutorials'.split()) # Skip these during the build 12 13def pathsplit(pkg_dir, path): 14 """Recursively split a path, returns a tuple""" 15 stem, basename = os.path.split(path) 16 if stem == '' or stem == pkg_dir: 17 return (basename,) 18 if stem == path: # fixed point, likely '/' 19 return (None,) 20 return pathsplit(pkg_dir, stem) + (basename,) 21 22def getlangext(name): 23 """Returns everything after the first . in the filename, including the .""" 24 file = os.path.basename(name) 25 loc = file.find('.') 26 if loc > -1: return file[loc:] 27 else: return '' 28 29def getlangsplit(name): 30 """Returns everything before the first . in the filename, excluding the .""" 31 file = os.path.basename(name) 32 loc = file.find('.') 33 if loc > -1: return os.path.join(os.path.dirname(name),file[:loc]) 34 raise RuntimeError("No . in filename") 35 36class Mistakes(object): 37 def __init__(self, log, verbose=False): 38 self.mistakes = [] 39 self.verbose = verbose 40 self.log = log 41 42 def compareDirLists(self,root, mdirs, dirs): 43 if SKIPDIRS.intersection(pathsplit(None, root)): 44 return 45 smdirs = set(mdirs).difference(AUTODIRS) 46 sdirs = set(dirs).difference(AUTODIRS) 47 if not smdirs.issubset(sdirs): 48 self.mistakes.append('%s/makefile contains a directory not on the filesystem: %r' % (root, sorted(smdirs - sdirs))) 49 if not self.verbose: return 50 if smdirs != sdirs: 51 from sys import stderr 52 stderr.write('Directory mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 53 % (root, 54 'in makefile ',sorted(smdirs), 55 'on filesystem ',sorted(sdirs), 56 'symmetric diff',sorted(smdirs.symmetric_difference(sdirs)))) 57 58 def summary(self): 59 for m in self.mistakes: 60 self.log.write(m + '\n') 61 if self.mistakes: 62 raise RuntimeError('\n\nThe PETSc makefiles contain mistakes or files are missing on the filesystem.\n%s\nPossible reasons:\n\t1. Files were deleted locally, try "git checkout filename", where "filename" is the missing file.\n\t2. Files were deleted from the repository, but were not removed from the makefile. Send mail to petsc-maint@mcs.anl.gov.\n\t3. Someone forgot to "add" new files to the repository. Send mail to petsc-maint@mcs.anl.gov.\n\n' % ('\n'.join(self.mistakes))) 63 64def stripsplit(line): 65 return line[len('#requires'):].replace("'","").split() 66 67PetscPKGS = 'sys vec mat dm ksp snes ts tao'.split() 68# the key is actually the language suffix, it won't work for suffixes such as 'kokkos.cxx' so use an _ and replace the _ as needed with . 69LANGS = dict(kokkos_cxx='KOKKOS', hip_cpp='HIP', sycl_cxx='SYCL', raja_cxx='RAJA', c='C', cxx='CXX', cpp='CPP', cu='CU', F='F', F90='F90') 70 71class debuglogger(object): 72 def __init__(self, log): 73 self._log = log 74 75 def write(self, string): 76 self._log.debug(string) 77 78class Petsc(object): 79 def __init__(self, petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, verbose=False): 80 if petsc_dir is None: 81 petsc_dir = os.environ.get('PETSC_DIR') 82 if petsc_dir is None: 83 try: 84 petsc_dir = parse_makefile(os.path.join('lib','petsc','conf', 'petscvariables')).get('PETSC_DIR') 85 finally: 86 if petsc_dir is None: 87 raise RuntimeError('Could not determine PETSC_DIR, please set in environment') 88 if petsc_arch is None: 89 petsc_arch = os.environ.get('PETSC_ARCH') 90 if petsc_arch is None: 91 try: 92 petsc_arch = parse_makefile(os.path.join(petsc_dir, 'lib','petsc','conf', 'petscvariables')).get('PETSC_ARCH') 93 finally: 94 if petsc_arch is None: 95 raise RuntimeError('Could not determine PETSC_ARCH, please set in environment') 96 self.petsc_dir = os.path.normpath(petsc_dir) 97 self.petsc_arch = petsc_arch.rstrip(os.sep) 98 self.pkg_dir = pkg_dir 99 self.pkg_name = pkg_name 100 self.pkg_arch = pkg_arch 101 if self.pkg_dir is None: 102 self.pkg_dir = petsc_dir 103 self.pkg_name = 'petsc' 104 self.pkg_arch = self.petsc_arch 105 if self.pkg_name is None: 106 self.pkg_name = os.path.basename(os.path.normpath(self.pkg_dir)) 107 if self.pkg_arch is None: 108 self.pkg_arch = self.petsc_arch 109 self.pkg_pkgs = PetscPKGS 110 if pkg_pkgs is not None: 111 if pkg_pkgs.find(',') > 0: npkgs = set(pkg_pkgs.split(',')) 112 else: npkgs = set(pkg_pkgs.split(' ')) 113 self.pkg_pkgs += list(npkgs - set(self.pkg_pkgs)) 114 self.read_conf() 115 try: 116 logging.basicConfig(filename=self.pkg_arch_path('lib',self.pkg_name,'conf', 'gmake.log'), level=logging.DEBUG) 117 except IOError: 118 # Disable logging if path is not writeable (e.g., prefix install) 119 logging.basicConfig(filename='/dev/null', level=logging.DEBUG) 120 self.log = logging.getLogger('gmakegen') 121 self.mistakes = Mistakes(debuglogger(self.log), verbose=verbose) 122 self.gendeps = [] 123 124 def arch_path(self, *args): 125 return os.path.join(self.petsc_dir, self.petsc_arch, *args) 126 127 def pkg_arch_path(self, *args): 128 return os.path.join(self.pkg_dir, self.pkg_arch, *args) 129 130 def read_conf(self): 131 self.conf = dict() 132 with open(self.arch_path('include', 'petscconf.h')) as petscconf_h: 133 for line in petscconf_h: 134 if line.startswith('#define '): 135 define = line[len('#define '):] 136 space = define.find(' ') 137 key = define[:space] 138 val = define[space+1:] 139 self.conf[key] = val 140 self.conf.update(parse_makefile(self.arch_path('lib','petsc','conf', 'petscvariables'))) 141 # allow parsing package additional configurations (if any) 142 if self.pkg_name != 'petsc' : 143 f = self.pkg_arch_path('include', self.pkg_name + 'conf.h') 144 if os.path.isfile(f): 145 with open(self.pkg_arch_path('include', self.pkg_name + 'conf.h')) as pkg_conf_h: 146 for line in pkg_conf_h: 147 if line.startswith('#define '): 148 define = line[len('#define '):] 149 space = define.find(' ') 150 key = define[:space] 151 val = define[space+1:] 152 self.conf[key] = val 153 f = self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables') 154 if os.path.isfile(f): 155 self.conf.update(parse_makefile(self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables'))) 156 self.have_fortran = int(self.conf.get('PETSC_USE_FORTRAN_BINDINGS', '0')) 157 158 def inconf(self, key, val): 159 if key in ['package', 'function', 'define']: 160 return self.conf.get(val) 161 elif key == 'precision': 162 return val == self.conf['PETSC_PRECISION'] 163 elif key == 'scalar': 164 return val == self.conf['PETSC_SCALAR'] 165 elif key == 'language': 166 return val == self.conf['PETSC_LANGUAGE'] 167 raise RuntimeError('Unknown conf check: %s %s' % (key, val)) 168 169 def relpath(self, root, src): 170 return os.path.relpath(os.path.join(root, src), self.pkg_dir) 171 172 def get_sources_from_files(self, files): 173 """Return dict {lang: list_of_source_files}""" 174 source = dict() 175 for lang, sourcelang in LANGS.items(): 176 source[lang] = [f for f in files if f.endswith('.'+lang.replace('_','.'))] 177 files = [f for f in files if not f.endswith('.'+lang.replace('_','.'))] 178 return source 179 180 def gen_pkg(self, pkg): 181 from itertools import chain 182 pkgsrcs = dict() 183 for lang in LANGS: 184 pkgsrcs[lang] = [] 185 for root, dirs, files in chain.from_iterable(os.walk(path) for path in [os.path.join(self.pkg_dir, 'src', pkg),os.path.join(self.pkg_dir, self.pkg_arch, 'src', pkg)]): 186 if SKIPDIRS.intersection(pathsplit(self.pkg_dir, root)): continue 187 dirs.sort() 188 files.sort() 189 makefile = os.path.join(root,'makefile') 190 if os.path.isfile(makefile): 191 with open(makefile) as mklines: 192 conditions = set(tuple(stripsplit(line)) for line in mklines if line.startswith('#requires')) 193 if not all(self.inconf(key, val) for key, val in conditions): 194 dirs[:] = [] 195 continue 196 makevars = parse_makefile(makefile) 197 mdirs = makevars.get('DIRS','').split() # Directories specified in the makefile 198 self.mistakes.compareDirLists(root, mdirs, dirs) # diagnostic output to find unused directories 199 candidates = set(mdirs).union(AUTODIRS).difference(SKIPDIRS) 200 dirs[:] = list(candidates.intersection(dirs)) 201 allsource = [] 202 def mkrel(src): 203 return self.relpath(root, src) 204 if files: 205 source = self.get_sources_from_files(files) 206 for lang, s in source.items(): 207 pkgsrcs[lang] += [mkrel(t) for t in s] 208 if os.path.isfile(makefile): self.gendeps.append(self.relpath(root, 'makefile')) 209 return pkgsrcs 210 211 def gen_gnumake(self, fd): 212 def write(stem, srcs): 213 for lang in LANGS: 214 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]))) 215 for pkg in self.pkg_pkgs: 216 srcs = self.gen_pkg(pkg) 217 write('srcs-' + pkg, srcs) 218 return self.gendeps 219 220 def gen_ninja(self, fd): 221 libobjs = [] 222 for pkg in self.pkg_pkgs: 223 srcs = self.gen_pkg(pkg) 224 for lang in LANGS: 225 for src in srcs[lang]: 226 obj = '$objdir/%s.o' % src 227 fd.write('build %(obj)s : %(lang)s_COMPILE %(src)s\n' % dict(obj=obj, lang=lang.upper(), src=os.path.join(self.pkg_dir,src))) 228 libobjs.append(obj) 229 fd.write('\n') 230 fd.write('build $libdir/libpetsc.so : %s_LINK_SHARED %s\n\n' % ('CF'[self.have_fortran], ' '.join(libobjs))) 231 fd.write('build petsc : phony || $libdir/libpetsc.so\n\n') 232 233 def summary(self): 234 self.mistakes.summary() 235 236def WriteGnuMake(petsc): 237 arch_files = petsc.pkg_arch_path('lib',petsc.pkg_name,'conf', 'files') 238 with open(arch_files, 'w') as fd: 239 gendeps = petsc.gen_gnumake(fd) 240 fd.write('\n') 241 fd.write('# Dependency to regenerate this file\n') 242 fd.write('%s : %s %s\n' % (os.path.relpath(arch_files, petsc.pkg_dir), 243 os.path.relpath(__file__, os.path.realpath(petsc.pkg_dir)), 244 ' '.join(gendeps))) 245 fd.write('\n') 246 fd.write('# Dummy dependencies in case makefiles are removed\n') 247 fd.write(''.join([dep + ':\n' for dep in gendeps])) 248 249def WriteNinja(petsc): 250 conf = dict() 251 parse_makefile(os.path.join(petsc.petsc_dir, 'lib', 'petsc','conf', 'variables'), conf) 252 parse_makefile(petsc.arch_path('lib','petsc','conf', 'petscvariables'), conf) 253 build_ninja = petsc.arch_path('build.ninja') 254 with open(build_ninja, 'w') as fd: 255 fd.write('objdir = obj-ninja\n') 256 fd.write('libdir = lib\n') 257 fd.write('c_compile = %(PCC)s\n' % conf) 258 fd.write('c_flags = %(PETSC_CC_INCLUDES)s %(PCC_FLAGS)s %(CCPPFLAGS)s\n' % conf) 259 fd.write('c_link = %(PCC_LINKER)s\n' % conf) 260 fd.write('c_link_flags = %(PCC_LINKER_FLAGS)s\n' % conf) 261 if petsc.have_fortran: 262 fd.write('f_compile = %(FC)s\n' % conf) 263 fd.write('f_flags = %(PETSC_FC_INCLUDES)s %(FC_FLAGS)s %(FCPPFLAGS)s\n' % conf) 264 fd.write('f_link = %(FC_LINKER)s\n' % conf) 265 fd.write('f_link_flags = %(FC_LINKER_FLAGS)s\n' % conf) 266 fd.write('petsc_external_lib = %(PETSC_EXTERNAL_LIB_BASIC)s\n' % conf) 267 fd.write('python = %(PYTHON)s\n' % conf) 268 fd.write('\n') 269 fd.write('rule C_COMPILE\n' 270 ' command = $c_compile -MMD -MF $out.d $c_flags -c $in -o $out\n' 271 ' description = CC $out\n' 272 ' depfile = $out.d\n' 273 # ' deps = gcc\n') # 'gcc' is default, 'msvc' only recognized by newer versions of ninja 274 '\n') 275 fd.write('rule C_LINK_SHARED\n' 276 ' command = $c_link $c_link_flags -shared -o $out $in $petsc_external_lib\n' 277 ' description = CLINK_SHARED $out\n' 278 '\n') 279 if petsc.have_fortran: 280 fd.write('rule F_COMPILE\n' 281 ' command = $f_compile -MMD -MF $out.d $f_flags -c $in -o $out\n' 282 ' description = FC $out\n' 283 ' depfile = $out.d\n' 284 '\n') 285 fd.write('rule F_LINK_SHARED\n' 286 ' command = $f_link $f_link_flags -shared -o $out $in $petsc_external_lib\n' 287 ' description = FLINK_SHARED $out\n' 288 '\n') 289 fd.write('rule GEN_NINJA\n' 290 ' command = $python $in --output=ninja\n' 291 ' generator = 1\n' 292 '\n') 293 petsc.gen_ninja(fd) 294 fd.write('\n') 295 fd.write('build %s : GEN_NINJA | %s %s %s %s\n' % (build_ninja, 296 os.path.abspath(__file__), 297 os.path.join(petsc.petsc_dir, 'lib','petsc','conf', 'variables'), 298 petsc.arch_path('lib','petsc','conf', 'petscvariables'), 299 ' '.join(os.path.join(petsc.pkg_dir, dep) for dep in petsc.gendeps))) 300 301def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, output=None, verbose=False): 302 if output is None: 303 output = 'gnumake' 304 writer = dict(gnumake=WriteGnuMake, ninja=WriteNinja) 305 petsc = Petsc(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_name=pkg_name, pkg_arch=pkg_arch, pkg_pkgs=pkg_pkgs, verbose=verbose) 306 writer[output](petsc) 307 petsc.summary() 308 309if __name__ == '__main__': 310 import optparse 311 parser = optparse.OptionParser() 312 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 313 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 314 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 315 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 316 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 317 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 318 parser.add_option('--output', help='Location to write output file', default=None) 319 opts, extra_args = parser.parse_args() 320 if extra_args: 321 import sys 322 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 323 exit(1) 324 main(petsc_arch=opts.petsc_arch, pkg_dir=opts.pkg_dir, pkg_name=opts.pkg_name, pkg_arch=opts.pkg_arch, pkg_pkgs=opts.pkg_pkgs, output=opts.output, verbose=opts.verbose) 325