1#!/usr/bin/env python 2 3import os 4from sysconfig import _parse_makefile as parse_makefile 5import sys 6import logging 7sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 8from collections import defaultdict 9 10AUTODIRS = set('ftn-auto ftn-custom f90-custom'.split()) # Automatically recurse into these, if they exist 11SKIPDIRS = set('benchmarks build'.split()) # Skip these during the build 12NOWARNDIRS = set('tests tutorials'.split()) # Do not warn about mismatch in these 13 14def pathsplit(path): 15 """Recursively split a path, returns a tuple""" 16 stem, basename = os.path.split(path) 17 if stem == '': 18 return (basename,) 19 if stem == path: # fixed point, likely '/' 20 return (path,) 21 return pathsplit(stem) + (basename,) 22 23def getlangext(name): 24 """Returns everything after the first . in the filename, including the .""" 25 file = os.path.basename(name) 26 loc = file.find('.') 27 if loc > -1: return file[loc:] 28 else: return '' 29 30def getlangsplit(name): 31 """Returns everything before the first . in the filename, excluding the .""" 32 file = os.path.basename(name) 33 loc = file.find('.') 34 if loc > -1: return os.path.join(os.path.dirname(name),file[:loc]) 35 raise RuntimeError("No . in filename") 36 37class Mistakes(object): 38 def __init__(self, log, verbose=False): 39 self.mistakes = [] 40 self.verbose = verbose 41 self.log = log 42 43 def compareDirLists(self,root, mdirs, dirs): 44 if NOWARNDIRS.intersection(pathsplit(root)): 45 return 46 smdirs = set(mdirs) 47 sdirs = set(dirs).difference(AUTODIRS) 48 if not smdirs.issubset(sdirs): 49 self.mistakes.append('Makefile contains directory not on filesystem: %s: %r' % (root, sorted(smdirs - sdirs))) 50 if not self.verbose: return 51 if smdirs != sdirs: 52 from sys import stderr 53 stderr.write('Directory mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 54 % (root, 55 'in makefile ',sorted(smdirs), 56 'on filesystem ',sorted(sdirs), 57 'symmetric diff',sorted(smdirs.symmetric_difference(sdirs)))) 58 59 def compareSourceLists(self, root, msources, files): 60 if NOWARNDIRS.intersection(pathsplit(root)): 61 return 62 smsources = set(msources) 63 ssources = set(f for f in files if getlangext(f) in ['.c', '.kokkos.cxx','.cxx', '.cc', '.cu', '.cpp', '.F', '.F90', '.hip.cpp', '.sycl.cxx']) 64 if not smsources.issubset(ssources): 65 self.mistakes.append('Makefile contains file not on filesystem: %s: %r' % (root, sorted(smsources - ssources))) 66 if not self.verbose: return 67 if smsources != ssources: 68 from sys import stderr 69 stderr.write('Source mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 70 % (root, 71 'in makefile ',sorted(smsources), 72 'on filesystem ',sorted(ssources), 73 'symmetric diff',sorted(smsources.symmetric_difference(ssources)))) 74 75 def summary(self): 76 for m in self.mistakes: 77 self.log.write(m + '\n') 78 if self.mistakes: 79 raise RuntimeError('PETSc makefiles contain mistakes or files are missing on filesystem.\n%s\nPossible reasons:\n\t1. Files were deleted locally, try "hg revert filename" or "git checkout filename".\n\t2. Files were deleted from repository, but were not removed from makefile. Send mail to petsc-maint@mcs.anl.gov.\n\t3. Someone forgot to "add" new files to the repository. Send mail to petsc-maint@mcs.anl.gov.' % ('\n'.join(self.mistakes))) 80 81def stripsplit(line): 82 return line[len('#requires'):].replace("'","").split() 83 84PetscPKGS = 'sys vec mat dm ksp snes ts tao'.split() 85# the key is actually the language suffix, it won't work for suffixes such as 'kokkos.cxx' so use an _ and replace the _ as needed with . 86LANGS = dict(kokkos_cxx='KOKKOS', c='C', cxx='CXX', cpp='CPP', cu='CU', F='F', 87 F90='F90', hip_cpp='HIP', sycl_cxx='SYCL.CXX') 88 89class debuglogger(object): 90 def __init__(self, log): 91 self._log = log 92 93 def write(self, string): 94 self._log.debug(string) 95 96class Petsc(object): 97 def __init__(self, petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, verbose=False): 98 if petsc_dir is None: 99 petsc_dir = os.environ.get('PETSC_DIR') 100 if petsc_dir is None: 101 try: 102 petsc_dir = parse_makefile(os.path.join('lib','petsc','conf', 'petscvariables')).get('PETSC_DIR') 103 finally: 104 if petsc_dir is None: 105 raise RuntimeError('Could not determine PETSC_DIR, please set in environment') 106 if petsc_arch is None: 107 petsc_arch = os.environ.get('PETSC_ARCH') 108 if petsc_arch is None: 109 try: 110 petsc_arch = parse_makefile(os.path.join(petsc_dir, 'lib','petsc','conf', 'petscvariables')).get('PETSC_ARCH') 111 finally: 112 if petsc_arch is None: 113 raise RuntimeError('Could not determine PETSC_ARCH, please set in environment') 114 self.petsc_dir = os.path.normpath(petsc_dir) 115 self.petsc_arch = petsc_arch.rstrip(os.sep) 116 self.pkg_dir = pkg_dir 117 self.pkg_name = pkg_name 118 self.pkg_arch = pkg_arch 119 if self.pkg_dir is None: 120 self.pkg_dir = petsc_dir 121 self.pkg_name = 'petsc' 122 self.pkg_arch = self.petsc_arch 123 if self.pkg_name is None: 124 self.pkg_name = os.path.basename(os.path.normpath(self.pkg_dir)) 125 if self.pkg_arch is None: 126 self.pkg_arch = self.petsc_arch 127 self.pkg_pkgs = PetscPKGS 128 if pkg_pkgs is not None: 129 self.pkg_pkgs += list(set(pkg_pkgs.split(','))-set(self.pkg_pkgs)) 130 self.read_conf() 131 try: 132 logging.basicConfig(filename=self.pkg_arch_path('lib',self.pkg_name,'conf', 'gmake.log'), level=logging.DEBUG) 133 except IOError: 134 # Disable logging if path is not writeable (e.g., prefix install) 135 logging.basicConfig(filename='/dev/null', level=logging.DEBUG) 136 self.log = logging.getLogger('gmakegen') 137 self.mistakes = Mistakes(debuglogger(self.log), verbose=verbose) 138 self.gendeps = [] 139 140 def arch_path(self, *args): 141 return os.path.join(self.petsc_dir, self.petsc_arch, *args) 142 143 def pkg_arch_path(self, *args): 144 return os.path.join(self.pkg_dir, self.pkg_arch, *args) 145 146 def read_conf(self): 147 self.conf = dict() 148 with open(self.arch_path('include', 'petscconf.h')) as petscconf_h: 149 for line in petscconf_h: 150 if line.startswith('#define '): 151 define = line[len('#define '):] 152 space = define.find(' ') 153 key = define[:space] 154 val = define[space+1:] 155 self.conf[key] = val 156 self.conf.update(parse_makefile(self.arch_path('lib','petsc','conf', 'petscvariables'))) 157 # allow parsing package additional configurations (if any) 158 if self.pkg_name != 'petsc' : 159 f = self.pkg_arch_path('include', self.pkg_name + 'conf.h') 160 if os.path.isfile(f): 161 with open(self.pkg_arch_path('include', self.pkg_name + 'conf.h')) as pkg_conf_h: 162 for line in pkg_conf_h: 163 if line.startswith('#define '): 164 define = line[len('#define '):] 165 space = define.find(' ') 166 key = define[:space] 167 val = define[space+1:] 168 self.conf[key] = val 169 f = self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables') 170 if os.path.isfile(f): 171 self.conf.update(parse_makefile(self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables'))) 172 self.have_fortran = int(self.conf.get('PETSC_HAVE_FORTRAN', '0')) 173 174 def inconf(self, key, val): 175 if key in ['package', 'function', 'define']: 176 return self.conf.get(val) 177 elif key == 'precision': 178 return val == self.conf['PETSC_PRECISION'] 179 elif key == 'scalar': 180 return val == self.conf['PETSC_SCALAR'] 181 elif key == 'language': 182 return val == self.conf['PETSC_LANGUAGE'] 183 raise RuntimeError('Unknown conf check: %s %s' % (key, val)) 184 185 def relpath(self, root, src): 186 return os.path.relpath(os.path.join(root, src), self.pkg_dir) 187 188 def get_sources(self, makevars): 189 """Return dict {lang: list_of_source_files}""" 190 source = dict() 191 for lang, sourcelang in LANGS.items(): 192 source[lang] = [f for f in makevars.get('SOURCE'+sourcelang,'').split() if f.endswith(lang.replace('_','.'))] 193 194 #source[lang] = [f for f in makevars.get('SOURCE'+sourcelang,'').split() if f.endswith(lang)] 195#SEK source[lang] = [f for f in 196#SEK makevars.get('SOURCE'+sourcelang,'').split() if 197#SEK f.endswith(LANGSEXT[lang])] 198 return source 199 200 def gen_pkg(self, pkg): 201 pkgsrcs = dict() 202 for lang in LANGS: 203 pkgsrcs[lang] = [] 204 for root, dirs, files in os.walk(os.path.join(self.pkg_dir, 'src', pkg)): 205 dirs.sort() 206 files.sort() 207 makefile = os.path.join(root,'makefile') 208 if not os.path.exists(makefile): 209 dirs[:] = [] 210 continue 211 with open(makefile) as mklines: 212 conditions = set(tuple(stripsplit(line)) for line in mklines if line.startswith('#requires')) 213 if not all(self.inconf(key, val) for key, val in conditions): 214 dirs[:] = [] 215 continue 216 makevars = parse_makefile(makefile) 217 mdirs = makevars.get('DIRS','').split() # Directories specified in the makefile 218 self.mistakes.compareDirLists(root, mdirs, dirs) # diagnostic output to find unused directories 219 candidates = set(mdirs).union(AUTODIRS).difference(SKIPDIRS) 220 dirs[:] = list(candidates.intersection(dirs)) 221 allsource = [] 222 def mkrel(src): 223 return self.relpath(root, src) 224 source = self.get_sources(makevars) 225 for lang, s in source.items(): 226 pkgsrcs[lang] += [mkrel(t) for t in s] 227 allsource += s 228 self.mistakes.compareSourceLists(root, allsource, files) # Diagnostic output about unused source files 229 self.gendeps.append(self.relpath(root, 'makefile')) 230 return pkgsrcs 231 232 def gen_gnumake(self, fd): 233 def write(stem, srcs): 234 for lang in LANGS: 235 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]))) 236 for pkg in self.pkg_pkgs: 237 srcs = self.gen_pkg(pkg) 238 write('srcs-' + pkg, srcs) 239 return self.gendeps 240 241 def gen_ninja(self, fd): 242 libobjs = [] 243 for pkg in self.pkg_pkgs: 244 srcs = self.gen_pkg(pkg) 245 for lang in LANGS: 246 for src in srcs[lang]: 247 obj = '$objdir/%s.o' % src 248 fd.write('build %(obj)s : %(lang)s_COMPILE %(src)s\n' % dict(obj=obj, lang=lang.upper(), src=os.path.join(self.pkg_dir,src))) 249 libobjs.append(obj) 250 fd.write('\n') 251 fd.write('build $libdir/libpetsc.so : %s_LINK_SHARED %s\n\n' % ('CF'[self.have_fortran], ' '.join(libobjs))) 252 fd.write('build petsc : phony || $libdir/libpetsc.so\n\n') 253 254 def summary(self): 255 self.mistakes.summary() 256 257def WriteGnuMake(petsc): 258 arch_files = petsc.pkg_arch_path('lib',petsc.pkg_name,'conf', 'files') 259 with open(arch_files, 'w') as fd: 260 gendeps = petsc.gen_gnumake(fd) 261 fd.write('\n') 262 fd.write('# Dependency to regenerate this file\n') 263 fd.write('%s : %s %s\n' % (os.path.relpath(arch_files, petsc.pkg_dir), 264 os.path.relpath(__file__, os.path.realpath(petsc.pkg_dir)), 265 ' '.join(gendeps))) 266 fd.write('\n') 267 fd.write('# Dummy dependencies in case makefiles are removed\n') 268 fd.write(''.join([dep + ':\n' for dep in gendeps])) 269 270def WriteNinja(petsc): 271 conf = dict() 272 parse_makefile(os.path.join(petsc.petsc_dir, 'lib', 'petsc','conf', 'variables'), conf) 273 parse_makefile(petsc.arch_path('lib','petsc','conf', 'petscvariables'), conf) 274 build_ninja = petsc.arch_path('build.ninja') 275 with open(build_ninja, 'w') as fd: 276 fd.write('objdir = obj-ninja\n') 277 fd.write('libdir = lib\n') 278 fd.write('c_compile = %(PCC)s\n' % conf) 279 fd.write('c_flags = %(PETSC_CC_INCLUDES)s %(PCC_FLAGS)s %(CCPPFLAGS)s\n' % conf) 280 fd.write('c_link = %(PCC_LINKER)s\n' % conf) 281 fd.write('c_link_flags = %(PCC_LINKER_FLAGS)s\n' % conf) 282 if petsc.have_fortran: 283 fd.write('f_compile = %(FC)s\n' % conf) 284 fd.write('f_flags = %(PETSC_FC_INCLUDES)s %(FC_FLAGS)s %(FCPPFLAGS)s\n' % conf) 285 fd.write('f_link = %(FC_LINKER)s\n' % conf) 286 fd.write('f_link_flags = %(FC_LINKER_FLAGS)s\n' % conf) 287 fd.write('petsc_external_lib = %(PETSC_EXTERNAL_LIB_BASIC)s\n' % conf) 288 fd.write('python = %(PYTHON)s\n' % conf) 289 fd.write('\n') 290 fd.write('rule C_COMPILE\n' 291 ' command = $c_compile -MMD -MF $out.d $c_flags -c $in -o $out\n' 292 ' description = CC $out\n' 293 ' depfile = $out.d\n' 294 # ' deps = gcc\n') # 'gcc' is default, 'msvc' only recognized by newer versions of ninja 295 '\n') 296 fd.write('rule C_LINK_SHARED\n' 297 ' command = $c_link $c_link_flags -shared -o $out $in $petsc_external_lib\n' 298 ' description = CLINK_SHARED $out\n' 299 '\n') 300 if petsc.have_fortran: 301 fd.write('rule F_COMPILE\n' 302 ' command = $f_compile -MMD -MF $out.d $f_flags -c $in -o $out\n' 303 ' description = FC $out\n' 304 ' depfile = $out.d\n' 305 '\n') 306 fd.write('rule F_LINK_SHARED\n' 307 ' command = $f_link $f_link_flags -shared -o $out $in $petsc_external_lib\n' 308 ' description = FLINK_SHARED $out\n' 309 '\n') 310 fd.write('rule GEN_NINJA\n' 311 ' command = $python $in --output=ninja\n' 312 ' generator = 1\n' 313 '\n') 314 petsc.gen_ninja(fd) 315 fd.write('\n') 316 fd.write('build %s : GEN_NINJA | %s %s %s %s\n' % (build_ninja, 317 os.path.abspath(__file__), 318 os.path.join(petsc.petsc_dir, 'lib','petsc','conf', 'variables'), 319 petsc.arch_path('lib','petsc','conf', 'petscvariables'), 320 ' '.join(os.path.join(petsc.pkg_dir, dep) for dep in petsc.gendeps))) 321 322def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, output=None, verbose=False): 323 if output is None: 324 output = 'gnumake' 325 writer = dict(gnumake=WriteGnuMake, ninja=WriteNinja) 326 petsc = Petsc(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_name=pkg_name, pkg_arch=pkg_arch, pkg_pkgs=pkg_pkgs, verbose=verbose) 327 writer[output](petsc) 328 petsc.summary() 329 330if __name__ == '__main__': 331 import optparse 332 parser = optparse.OptionParser() 333 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 334 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 335 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 336 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 337 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 338 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 339 parser.add_option('--output', help='Location to write output file', default=None) 340 opts, extra_args = parser.parse_args() 341 if extra_args: 342 import sys 343 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 344 exit(1) 345 main(petsc_arch=opts.petsc_arch, pkg_dir=opts.pkg_dir, pkg_name=opts.pkg_name, pkg_arch=opts.pkg_arch, pkg_pkgs=opts.pkg_pkgs, output=opts.output, verbose=opts.verbose) 346