1#!/usr/bin/env python 2 3import os 4from sysconfig import _parse_makefile as parse_makefile 5import sys 6import logging 7sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 8from collections import defaultdict 9 10AUTODIRS = set('ftn-auto ftn-custom f90-custom'.split()) # Automatically recurse into these, if they exist 11SKIPDIRS = set('benchmarks build'.split()) # Skip these during the build 12NOWARNDIRS = set('tests tutorials'.split()) # Do not warn about mismatch in these 13 14def pathsplit(path): 15 """Recursively split a path, returns a tuple""" 16 stem, basename = os.path.split(path) 17 if stem == '': 18 return (basename,) 19 if stem == path: # fixed point, likely '/' 20 return (path,) 21 return pathsplit(stem) + (basename,) 22 23def getlangext(name): 24 """Returns everything after the first . in the filename, including the .""" 25 file = os.path.basename(name) 26 loc = file.find('.') 27 if loc > -1: return file[loc:] 28 else: return '' 29 30def getlangsplit(name): 31 """Returns everything before the first . in the filename, excluding the .""" 32 file = os.path.basename(name) 33 loc = file.find('.') 34 if loc > -1: return os.path.join(os.path.dirname(name),file[:loc]) 35 raise RuntimeError("No . in filename") 36 37class Mistakes(object): 38 def __init__(self, log, verbose=False): 39 self.mistakes = [] 40 self.verbose = verbose 41 self.log = log 42 43 def compareDirLists(self,root, mdirs, dirs): 44 if NOWARNDIRS.intersection(pathsplit(root)): 45 return 46 smdirs = set(mdirs) 47 sdirs = set(dirs).difference(AUTODIRS) 48 if not smdirs.issubset(sdirs): 49 self.mistakes.append('%s/makefile contains a directory not on the filesystem: %r' % (root, sorted(smdirs - sdirs))) 50 if not self.verbose: return 51 if smdirs != sdirs: 52 from sys import stderr 53 stderr.write('Directory mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 54 % (root, 55 'in makefile ',sorted(smdirs), 56 'on filesystem ',sorted(sdirs), 57 'symmetric diff',sorted(smdirs.symmetric_difference(sdirs)))) 58 59 def compareSourceLists(self, root, msources, files): 60 if NOWARNDIRS.intersection(pathsplit(root)): 61 return 62 smsources = set(msources) 63 ssources = set(f for f in files if getlangext(f) in ['.c', '.kokkos.cxx','.cxx', '.cc', '.cu', '.cpp', '.F', '.F90', '.hip.cpp', '.sycl.cxx']) 64 if not smsources.issubset(ssources): 65 self.mistakes.append('%s/makefile contains a file not on the filesystem: %r' % (root, sorted(smsources - ssources))) 66 if not self.verbose: return 67 if smsources != ssources: 68 from sys import stderr 69 stderr.write('Source mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 70 % (root, 71 'in makefile ',sorted(smsources), 72 'on filesystem ',sorted(ssources), 73 'symmetric diff',sorted(smsources.symmetric_difference(ssources)))) 74 75 def summary(self): 76 for m in self.mistakes: 77 self.log.write(m + '\n') 78 if self.mistakes: 79 raise RuntimeError('\n\nThe PETSc makefiles contain mistakes or files are missing on the filesystem.\n%s\nPossible reasons:\n\t1. Files were deleted locally, try "git checkout filename", where "filename" is the missing file.\n\t2. Files were deleted from the repository, but were not removed from the makefile. Send mail to petsc-maint@mcs.anl.gov.\n\t3. Someone forgot to "add" new files to the repository. Send mail to petsc-maint@mcs.anl.gov.\n\n' % ('\n'.join(self.mistakes))) 80 81def stripsplit(line): 82 return line[len('#requires'):].replace("'","").split() 83 84PetscPKGS = 'sys vec mat dm ksp snes ts tao'.split() 85# the key is actually the language suffix, it won't work for suffixes such as 'kokkos.cxx' so use an _ and replace the _ as needed with . 86LANGS = dict(kokkos_cxx='KOKKOS', c='C', cxx='CXX', cpp='CPP', cu='CU', F='F', 87 F90='F90', hip_cpp='HIP', sycl_cxx='SYCL.CXX') 88 89class debuglogger(object): 90 def __init__(self, log): 91 self._log = log 92 93 def write(self, string): 94 self._log.debug(string) 95 96class Petsc(object): 97 def __init__(self, petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, verbose=False): 98 if petsc_dir is None: 99 petsc_dir = os.environ.get('PETSC_DIR') 100 if petsc_dir is None: 101 try: 102 petsc_dir = parse_makefile(os.path.join('lib','petsc','conf', 'petscvariables')).get('PETSC_DIR') 103 finally: 104 if petsc_dir is None: 105 raise RuntimeError('Could not determine PETSC_DIR, please set in environment') 106 if petsc_arch is None: 107 petsc_arch = os.environ.get('PETSC_ARCH') 108 if petsc_arch is None: 109 try: 110 petsc_arch = parse_makefile(os.path.join(petsc_dir, 'lib','petsc','conf', 'petscvariables')).get('PETSC_ARCH') 111 finally: 112 if petsc_arch is None: 113 raise RuntimeError('Could not determine PETSC_ARCH, please set in environment') 114 self.petsc_dir = os.path.normpath(petsc_dir) 115 self.petsc_arch = petsc_arch.rstrip(os.sep) 116 self.pkg_dir = pkg_dir 117 self.pkg_name = pkg_name 118 self.pkg_arch = pkg_arch 119 if self.pkg_dir is None: 120 self.pkg_dir = petsc_dir 121 self.pkg_name = 'petsc' 122 self.pkg_arch = self.petsc_arch 123 if self.pkg_name is None: 124 self.pkg_name = os.path.basename(os.path.normpath(self.pkg_dir)) 125 if self.pkg_arch is None: 126 self.pkg_arch = self.petsc_arch 127 self.pkg_pkgs = PetscPKGS 128 if pkg_pkgs is not None: 129 self.pkg_pkgs += list(set(pkg_pkgs.split(','))-set(self.pkg_pkgs)) 130 self.read_conf() 131 try: 132 logging.basicConfig(filename=self.pkg_arch_path('lib',self.pkg_name,'conf', 'gmake.log'), level=logging.DEBUG) 133 except IOError: 134 # Disable logging if path is not writeable (e.g., prefix install) 135 logging.basicConfig(filename='/dev/null', level=logging.DEBUG) 136 self.log = logging.getLogger('gmakegen') 137 self.mistakes = Mistakes(debuglogger(self.log), verbose=verbose) 138 self.gendeps = [] 139 140 def arch_path(self, *args): 141 return os.path.join(self.petsc_dir, self.petsc_arch, *args) 142 143 def pkg_arch_path(self, *args): 144 return os.path.join(self.pkg_dir, self.pkg_arch, *args) 145 146 def read_conf(self): 147 self.conf = dict() 148 with open(self.arch_path('include', 'petscconf.h')) as petscconf_h: 149 for line in petscconf_h: 150 if line.startswith('#define '): 151 define = line[len('#define '):] 152 space = define.find(' ') 153 key = define[:space] 154 val = define[space+1:] 155 self.conf[key] = val 156 self.conf.update(parse_makefile(self.arch_path('lib','petsc','conf', 'petscvariables'))) 157 # allow parsing package additional configurations (if any) 158 if self.pkg_name != 'petsc' : 159 f = self.pkg_arch_path('include', self.pkg_name + 'conf.h') 160 if os.path.isfile(f): 161 with open(self.pkg_arch_path('include', self.pkg_name + 'conf.h')) as pkg_conf_h: 162 for line in pkg_conf_h: 163 if line.startswith('#define '): 164 define = line[len('#define '):] 165 space = define.find(' ') 166 key = define[:space] 167 val = define[space+1:] 168 self.conf[key] = val 169 f = self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables') 170 if os.path.isfile(f): 171 self.conf.update(parse_makefile(self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables'))) 172 self.have_fortran = int(self.conf.get('PETSC_HAVE_FORTRAN', '0')) 173 174 def inconf(self, key, val): 175 if key in ['package', 'function', 'define']: 176 return self.conf.get(val) 177 elif key == 'precision': 178 return val == self.conf['PETSC_PRECISION'] 179 elif key == 'scalar': 180 return val == self.conf['PETSC_SCALAR'] 181 elif key == 'language': 182 return val == self.conf['PETSC_LANGUAGE'] 183 raise RuntimeError('Unknown conf check: %s %s' % (key, val)) 184 185 def relpath(self, root, src): 186 return os.path.relpath(os.path.join(root, src), self.pkg_dir) 187 188 def get_sources(self, makevars): 189 """Return dict {lang: list_of_source_files}""" 190 source = dict() 191 for lang, sourcelang in LANGS.items(): 192 source[lang] = [f for f in makevars.get('SOURCE'+sourcelang,'').split() if f.endswith(lang.replace('_','.'))] 193 194 #source[lang] = [f for f in makevars.get('SOURCE'+sourcelang,'').split() if f.endswith(lang)] 195#SEK source[lang] = [f for f in 196#SEK makevars.get('SOURCE'+sourcelang,'').split() if 197#SEK f.endswith(LANGSEXT[lang])] 198 return source 199 200 def gen_pkg(self, pkg): 201 pkgsrcs = dict() 202 for lang in LANGS: 203 pkgsrcs[lang] = [] 204 for root, dirs, files in os.walk(os.path.join(self.pkg_dir, 'src', pkg)): 205 dirs.sort() 206 files.sort() 207 makefile = os.path.join(root,'makefile') 208 if not os.path.exists(makefile): 209 dirs[:] = [] 210 continue 211 with open(makefile) as mklines: 212 conditions = set(tuple(stripsplit(line)) for line in mklines if line.startswith('#requires')) 213 if not all(self.inconf(key, val) for key, val in conditions): 214 dirs[:] = [] 215 continue 216 makevars = parse_makefile(makefile) 217 mdirs = makevars.get('DIRS','').split() # Directories specified in the makefile 218 self.mistakes.compareDirLists(root, mdirs, dirs) # diagnostic output to find unused directories 219 candidates = set(mdirs).union(AUTODIRS).difference(SKIPDIRS) 220 dirs[:] = list(candidates.intersection(dirs)) 221 allsource = [] 222 def mkrel(src): 223 return self.relpath(root, src) 224 source = self.get_sources(makevars) 225 for lang, s in source.items(): 226 pkgsrcs[lang] += [mkrel(t) for t in s] 227 allsource += s 228 self.mistakes.compareSourceLists(root, allsource, files) # Diagnostic output about unused source files 229 self.gendeps.append(self.relpath(root, 'makefile')) 230 return pkgsrcs 231 232 def gen_gnumake(self, fd): 233 def write(stem, srcs): 234 for lang in LANGS: 235 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]))) 236 for pkg in self.pkg_pkgs: 237 srcs = self.gen_pkg(pkg) 238 write('srcs-' + pkg, srcs) 239 return self.gendeps 240 241 def gen_ninja(self, fd): 242 libobjs = [] 243 for pkg in self.pkg_pkgs: 244 srcs = self.gen_pkg(pkg) 245 for lang in LANGS: 246 for src in srcs[lang]: 247 obj = '$objdir/%s.o' % src 248 fd.write('build %(obj)s : %(lang)s_COMPILE %(src)s\n' % dict(obj=obj, lang=lang.upper(), src=os.path.join(self.pkg_dir,src))) 249 libobjs.append(obj) 250 fd.write('\n') 251 fd.write('build $libdir/libpetsc.so : %s_LINK_SHARED %s\n\n' % ('CF'[self.have_fortran], ' '.join(libobjs))) 252 fd.write('build petsc : phony || $libdir/libpetsc.so\n\n') 253 254 def summary(self): 255 self.mistakes.summary() 256 257def WriteGnuMake(petsc): 258 arch_files = petsc.pkg_arch_path('lib',petsc.pkg_name,'conf', 'files') 259 with open(arch_files, 'w') as fd: 260 gendeps = petsc.gen_gnumake(fd) 261 fd.write('\n') 262 fd.write('# Dependency to regenerate this file\n') 263 fd.write('%s : %s %s\n' % (os.path.relpath(arch_files, petsc.pkg_dir), 264 os.path.relpath(__file__, os.path.realpath(petsc.pkg_dir)), 265 ' '.join(gendeps))) 266 fd.write('\n') 267 fd.write('# Dummy dependencies in case makefiles are removed\n') 268 fd.write(''.join([dep + ':\n' for dep in gendeps])) 269 270def WriteNinja(petsc): 271 conf = dict() 272 parse_makefile(os.path.join(petsc.petsc_dir, 'lib', 'petsc','conf', 'variables'), conf) 273 parse_makefile(petsc.arch_path('lib','petsc','conf', 'petscvariables'), conf) 274 build_ninja = petsc.arch_path('build.ninja') 275 with open(build_ninja, 'w') as fd: 276 fd.write('objdir = obj-ninja\n') 277 fd.write('libdir = lib\n') 278 fd.write('c_compile = %(PCC)s\n' % conf) 279 fd.write('c_flags = %(PETSC_CC_INCLUDES)s %(PCC_FLAGS)s %(CCPPFLAGS)s\n' % conf) 280 fd.write('c_link = %(PCC_LINKER)s\n' % conf) 281 fd.write('c_link_flags = %(PCC_LINKER_FLAGS)s\n' % conf) 282 if petsc.have_fortran: 283 fd.write('f_compile = %(FC)s\n' % conf) 284 fd.write('f_flags = %(PETSC_FC_INCLUDES)s %(FC_FLAGS)s %(FCPPFLAGS)s\n' % conf) 285 fd.write('f_link = %(FC_LINKER)s\n' % conf) 286 fd.write('f_link_flags = %(FC_LINKER_FLAGS)s\n' % conf) 287 fd.write('petsc_external_lib = %(PETSC_EXTERNAL_LIB_BASIC)s\n' % conf) 288 fd.write('python = %(PYTHON)s\n' % conf) 289 fd.write('\n') 290 fd.write('rule C_COMPILE\n' 291 ' command = $c_compile -MMD -MF $out.d $c_flags -c $in -o $out\n' 292 ' description = CC $out\n' 293 ' depfile = $out.d\n' 294 # ' deps = gcc\n') # 'gcc' is default, 'msvc' only recognized by newer versions of ninja 295 '\n') 296 fd.write('rule C_LINK_SHARED\n' 297 ' command = $c_link $c_link_flags -shared -o $out $in $petsc_external_lib\n' 298 ' description = CLINK_SHARED $out\n' 299 '\n') 300 if petsc.have_fortran: 301 fd.write('rule F_COMPILE\n' 302 ' command = $f_compile -MMD -MF $out.d $f_flags -c $in -o $out\n' 303 ' description = FC $out\n' 304 ' depfile = $out.d\n' 305 '\n') 306 fd.write('rule F_LINK_SHARED\n' 307 ' command = $f_link $f_link_flags -shared -o $out $in $petsc_external_lib\n' 308 ' description = FLINK_SHARED $out\n' 309 '\n') 310 fd.write('rule GEN_NINJA\n' 311 ' command = $python $in --output=ninja\n' 312 ' generator = 1\n' 313 '\n') 314 petsc.gen_ninja(fd) 315 fd.write('\n') 316 fd.write('build %s : GEN_NINJA | %s %s %s %s\n' % (build_ninja, 317 os.path.abspath(__file__), 318 os.path.join(petsc.petsc_dir, 'lib','petsc','conf', 'variables'), 319 petsc.arch_path('lib','petsc','conf', 'petscvariables'), 320 ' '.join(os.path.join(petsc.pkg_dir, dep) for dep in petsc.gendeps))) 321 322def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, output=None, verbose=False): 323 if output is None: 324 output = 'gnumake' 325 writer = dict(gnumake=WriteGnuMake, ninja=WriteNinja) 326 petsc = Petsc(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_name=pkg_name, pkg_arch=pkg_arch, pkg_pkgs=pkg_pkgs, verbose=verbose) 327 writer[output](petsc) 328 petsc.summary() 329 330if __name__ == '__main__': 331 import optparse 332 parser = optparse.OptionParser() 333 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 334 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 335 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 336 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 337 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 338 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 339 parser.add_option('--output', help='Location to write output file', default=None) 340 opts, extra_args = parser.parse_args() 341 if extra_args: 342 import sys 343 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 344 exit(1) 345 main(petsc_arch=opts.petsc_arch, pkg_dir=opts.pkg_dir, pkg_name=opts.pkg_name, pkg_arch=opts.pkg_arch, pkg_pkgs=opts.pkg_pkgs, output=opts.output, verbose=opts.verbose) 346