1#!/usr/bin/env python3 2 3import os 4from sysconfig import _parse_makefile as parse_makefile 5import sys 6import logging 7sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 8from collections import defaultdict 9 10AUTODIRS = set('ftn-auto ftn-custom f90-custom ftn-auto-interfaces'.split()) # Automatically recurse into these, if they exist 11SKIPDIRS = set('benchmarks build mex-scripts tests tutorials'.split()) # Skip these during the build 12 13def pathsplit(pkg_dir, path): 14 """Recursively split a path, returns a tuple""" 15 stem, basename = os.path.split(path) 16 if stem == '' or stem == pkg_dir: 17 return (basename,) 18 if stem == path: # fixed point, likely '/' 19 return (None,) 20 return pathsplit(pkg_dir, stem) + (basename,) 21 22def getlangext(name): 23 """Returns everything after the first . in the filename, including the .""" 24 file = os.path.basename(name) 25 loc = file.find('.') 26 if loc > -1: return file[loc:] 27 else: return '' 28 29def getlangsplit(name): 30 """Returns everything before the first . in the filename, excluding the .""" 31 file = os.path.basename(name) 32 loc = file.find('.') 33 if loc > -1: return os.path.join(os.path.dirname(name),file[:loc]) 34 raise RuntimeError("No . in filename") 35 36class Mistakes(object): 37 def __init__(self, log, verbose=False): 38 self.mistakes = [] 39 self.verbose = verbose 40 self.log = log 41 42 def compareDirLists(self,root, mdirs, dirs): 43 if SKIPDIRS.intersection(pathsplit(None, root)): 44 return 45 smdirs = set(mdirs).difference(AUTODIRS) 46 sdirs = set(dirs).difference(AUTODIRS) 47 if not smdirs.issubset(sdirs): 48 self.mistakes.append('%s/makefile contains a directory not on the filesystem: %r' % (root, sorted(smdirs - sdirs))) 49 if not self.verbose: return 50 if smdirs != sdirs: 51 from sys import stderr 52 stderr.write('Directory mismatch at %s:\n\t%s: %r\n\t%s: %r\n\t%s: %r\n' 53 % (root, 54 'in makefile ',sorted(smdirs), 55 'on filesystem ',sorted(sdirs), 56 'symmetric diff',sorted(smdirs.symmetric_difference(sdirs)))) 57 58 def summary(self): 59 for m in self.mistakes: 60 self.log.write(m + '\n') 61 if self.mistakes: 62 raise RuntimeError('\n\nThe PETSc makefiles contain mistakes or files are missing on the filesystem.\n%s\nPossible reasons:\n\t1. Files were deleted locally, try "git checkout filename", where "filename" is the missing file.\n\t2. Files were deleted from the repository, but were not removed from the makefile. Send mail to petsc-maint@mcs.anl.gov.\n\t3. Someone forgot to "add" new files to the repository. Send mail to petsc-maint@mcs.anl.gov.\n\n' % ('\n'.join(self.mistakes))) 63 64def stripsplit(line): 65 return line[len('#requires'):].replace("'","").split() 66 67PetscPKGS = 'sys vec mat dm ksp snes ts tao'.split() 68# the key is actually the language suffix, it won't work for suffixes such as 'kokkos.cxx' so use an _ and replace the _ as needed with . 69LANGS = dict(kokkos_cxx='KOKKOS', hip_cpp='HIP', sycl_cxx='SYCL', raja_cxx='RAJA', c='C', cxx='CXX', cpp='CPP', cu='CU', F='F', F90='F90') 70 71class debuglogger(object): 72 def __init__(self, log): 73 self._log = log 74 75 def write(self, string): 76 self._log.debug(string) 77 78class Petsc(object): 79 def __init__(self, petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, verbose=False): 80 if petsc_dir is None: 81 petsc_dir = os.environ.get('PETSC_DIR') 82 if petsc_dir is None: 83 try: 84 petsc_dir = parse_makefile(os.path.join('lib','petsc','conf', 'petscvariables')).get('PETSC_DIR') 85 finally: 86 if petsc_dir is None: 87 raise RuntimeError('Could not determine PETSC_DIR, please set in environment') 88 if petsc_arch is None: 89 petsc_arch = os.environ.get('PETSC_ARCH') 90 if petsc_arch is None: 91 try: 92 petsc_arch = parse_makefile(os.path.join(petsc_dir, 'lib','petsc','conf', 'petscvariables')).get('PETSC_ARCH') 93 finally: 94 if petsc_arch is None: 95 raise RuntimeError('Could not determine PETSC_ARCH, please set in environment') 96 self.petsc_dir = os.path.normpath(petsc_dir) 97 self.petsc_arch = petsc_arch.rstrip(os.sep) 98 self.pkg_dir = pkg_dir 99 self.pkg_name = pkg_name 100 self.pkg_arch = pkg_arch 101 if self.pkg_dir is None: 102 self.pkg_dir = petsc_dir 103 self.pkg_name = 'petsc' 104 self.pkg_arch = self.petsc_arch 105 if self.pkg_name is None: 106 self.pkg_name = os.path.basename(os.path.normpath(self.pkg_dir)) 107 if self.pkg_arch is None: 108 self.pkg_arch = self.petsc_arch 109 self.pkg_pkgs = PetscPKGS 110 if pkg_pkgs is not None: 111 self.pkg_pkgs += list(set(pkg_pkgs.split(','))-set(self.pkg_pkgs)) 112 self.read_conf() 113 try: 114 logging.basicConfig(filename=self.pkg_arch_path('lib',self.pkg_name,'conf', 'gmake.log'), level=logging.DEBUG) 115 except IOError: 116 # Disable logging if path is not writeable (e.g., prefix install) 117 logging.basicConfig(filename='/dev/null', level=logging.DEBUG) 118 self.log = logging.getLogger('gmakegen') 119 self.mistakes = Mistakes(debuglogger(self.log), verbose=verbose) 120 self.gendeps = [] 121 122 def arch_path(self, *args): 123 return os.path.join(self.petsc_dir, self.petsc_arch, *args) 124 125 def pkg_arch_path(self, *args): 126 return os.path.join(self.pkg_dir, self.pkg_arch, *args) 127 128 def read_conf(self): 129 self.conf = dict() 130 with open(self.arch_path('include', 'petscconf.h')) as petscconf_h: 131 for line in petscconf_h: 132 if line.startswith('#define '): 133 define = line[len('#define '):] 134 space = define.find(' ') 135 key = define[:space] 136 val = define[space+1:] 137 self.conf[key] = val 138 self.conf.update(parse_makefile(self.arch_path('lib','petsc','conf', 'petscvariables'))) 139 # allow parsing package additional configurations (if any) 140 if self.pkg_name != 'petsc' : 141 f = self.pkg_arch_path('include', self.pkg_name + 'conf.h') 142 if os.path.isfile(f): 143 with open(self.pkg_arch_path('include', self.pkg_name + 'conf.h')) as pkg_conf_h: 144 for line in pkg_conf_h: 145 if line.startswith('#define '): 146 define = line[len('#define '):] 147 space = define.find(' ') 148 key = define[:space] 149 val = define[space+1:] 150 self.conf[key] = val 151 f = self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables') 152 if os.path.isfile(f): 153 self.conf.update(parse_makefile(self.pkg_arch_path('lib',self.pkg_name,'conf', self.pkg_name + 'variables'))) 154 self.have_fortran = int(self.conf.get('PETSC_HAVE_FORTRAN', '0')) 155 156 def inconf(self, key, val): 157 if key in ['package', 'function', 'define']: 158 return self.conf.get(val) 159 elif key == 'precision': 160 return val == self.conf['PETSC_PRECISION'] 161 elif key == 'scalar': 162 return val == self.conf['PETSC_SCALAR'] 163 elif key == 'language': 164 return val == self.conf['PETSC_LANGUAGE'] 165 raise RuntimeError('Unknown conf check: %s %s' % (key, val)) 166 167 def relpath(self, root, src): 168 return os.path.relpath(os.path.join(root, src), self.pkg_dir) 169 170 def get_sources_from_files(self, files): 171 """Return dict {lang: list_of_source_files}""" 172 source = dict() 173 for lang, sourcelang in LANGS.items(): 174 source[lang] = [f for f in files if f.endswith('.'+lang.replace('_','.'))] 175 files = [f for f in files if not f.endswith('.'+lang.replace('_','.'))] 176 return source 177 178 def gen_pkg(self, pkg): 179 pkgsrcs = dict() 180 for lang in LANGS: 181 pkgsrcs[lang] = [] 182 for root, dirs, files in os.walk(os.path.join(self.pkg_dir, 'src', pkg)): 183 if SKIPDIRS.intersection(pathsplit(self.pkg_dir, root)): continue 184 dirs.sort() 185 files.sort() 186 makefile = os.path.join(root,'makefile') 187 if not os.path.exists(makefile): 188 dirs[:] = [] 189 continue 190 with open(makefile) as mklines: 191 conditions = set(tuple(stripsplit(line)) for line in mklines if line.startswith('#requires')) 192 if not all(self.inconf(key, val) for key, val in conditions): 193 dirs[:] = [] 194 continue 195 makevars = parse_makefile(makefile) 196 mdirs = makevars.get('DIRS','').split() # Directories specified in the makefile 197 self.mistakes.compareDirLists(root, mdirs, dirs) # diagnostic output to find unused directories 198 candidates = set(mdirs).union(AUTODIRS).difference(SKIPDIRS) 199 dirs[:] = list(candidates.intersection(dirs)) 200 allsource = [] 201 def mkrel(src): 202 return self.relpath(root, src) 203 source = self.get_sources_from_files(files) 204 for lang, s in source.items(): 205 pkgsrcs[lang] += [mkrel(t) for t in s] 206 self.gendeps.append(self.relpath(root, 'makefile')) 207 return pkgsrcs 208 209 def gen_gnumake(self, fd): 210 def write(stem, srcs): 211 for lang in LANGS: 212 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]))) 213 for pkg in self.pkg_pkgs: 214 srcs = self.gen_pkg(pkg) 215 write('srcs-' + pkg, srcs) 216 return self.gendeps 217 218 def gen_ninja(self, fd): 219 libobjs = [] 220 for pkg in self.pkg_pkgs: 221 srcs = self.gen_pkg(pkg) 222 for lang in LANGS: 223 for src in srcs[lang]: 224 obj = '$objdir/%s.o' % src 225 fd.write('build %(obj)s : %(lang)s_COMPILE %(src)s\n' % dict(obj=obj, lang=lang.upper(), src=os.path.join(self.pkg_dir,src))) 226 libobjs.append(obj) 227 fd.write('\n') 228 fd.write('build $libdir/libpetsc.so : %s_LINK_SHARED %s\n\n' % ('CF'[self.have_fortran], ' '.join(libobjs))) 229 fd.write('build petsc : phony || $libdir/libpetsc.so\n\n') 230 231 def summary(self): 232 self.mistakes.summary() 233 234def WriteGnuMake(petsc): 235 arch_files = petsc.pkg_arch_path('lib',petsc.pkg_name,'conf', 'files') 236 with open(arch_files, 'w') as fd: 237 gendeps = petsc.gen_gnumake(fd) 238 fd.write('\n') 239 fd.write('# Dependency to regenerate this file\n') 240 fd.write('%s : %s %s\n' % (os.path.relpath(arch_files, petsc.pkg_dir), 241 os.path.relpath(__file__, os.path.realpath(petsc.pkg_dir)), 242 ' '.join(gendeps))) 243 fd.write('\n') 244 fd.write('# Dummy dependencies in case makefiles are removed\n') 245 fd.write(''.join([dep + ':\n' for dep in gendeps])) 246 247def WriteNinja(petsc): 248 conf = dict() 249 parse_makefile(os.path.join(petsc.petsc_dir, 'lib', 'petsc','conf', 'variables'), conf) 250 parse_makefile(petsc.arch_path('lib','petsc','conf', 'petscvariables'), conf) 251 build_ninja = petsc.arch_path('build.ninja') 252 with open(build_ninja, 'w') as fd: 253 fd.write('objdir = obj-ninja\n') 254 fd.write('libdir = lib\n') 255 fd.write('c_compile = %(PCC)s\n' % conf) 256 fd.write('c_flags = %(PETSC_CC_INCLUDES)s %(PCC_FLAGS)s %(CCPPFLAGS)s\n' % conf) 257 fd.write('c_link = %(PCC_LINKER)s\n' % conf) 258 fd.write('c_link_flags = %(PCC_LINKER_FLAGS)s\n' % conf) 259 if petsc.have_fortran: 260 fd.write('f_compile = %(FC)s\n' % conf) 261 fd.write('f_flags = %(PETSC_FC_INCLUDES)s %(FC_FLAGS)s %(FCPPFLAGS)s\n' % conf) 262 fd.write('f_link = %(FC_LINKER)s\n' % conf) 263 fd.write('f_link_flags = %(FC_LINKER_FLAGS)s\n' % conf) 264 fd.write('petsc_external_lib = %(PETSC_EXTERNAL_LIB_BASIC)s\n' % conf) 265 fd.write('python = %(PYTHON)s\n' % conf) 266 fd.write('\n') 267 fd.write('rule C_COMPILE\n' 268 ' command = $c_compile -MMD -MF $out.d $c_flags -c $in -o $out\n' 269 ' description = CC $out\n' 270 ' depfile = $out.d\n' 271 # ' deps = gcc\n') # 'gcc' is default, 'msvc' only recognized by newer versions of ninja 272 '\n') 273 fd.write('rule C_LINK_SHARED\n' 274 ' command = $c_link $c_link_flags -shared -o $out $in $petsc_external_lib\n' 275 ' description = CLINK_SHARED $out\n' 276 '\n') 277 if petsc.have_fortran: 278 fd.write('rule F_COMPILE\n' 279 ' command = $f_compile -MMD -MF $out.d $f_flags -c $in -o $out\n' 280 ' description = FC $out\n' 281 ' depfile = $out.d\n' 282 '\n') 283 fd.write('rule F_LINK_SHARED\n' 284 ' command = $f_link $f_link_flags -shared -o $out $in $petsc_external_lib\n' 285 ' description = FLINK_SHARED $out\n' 286 '\n') 287 fd.write('rule GEN_NINJA\n' 288 ' command = $python $in --output=ninja\n' 289 ' generator = 1\n' 290 '\n') 291 petsc.gen_ninja(fd) 292 fd.write('\n') 293 fd.write('build %s : GEN_NINJA | %s %s %s %s\n' % (build_ninja, 294 os.path.abspath(__file__), 295 os.path.join(petsc.petsc_dir, 'lib','petsc','conf', 'variables'), 296 petsc.arch_path('lib','petsc','conf', 'petscvariables'), 297 ' '.join(os.path.join(petsc.pkg_dir, dep) for dep in petsc.gendeps))) 298 299def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_name=None, pkg_arch=None, pkg_pkgs=None, output=None, verbose=False): 300 if output is None: 301 output = 'gnumake' 302 writer = dict(gnumake=WriteGnuMake, ninja=WriteNinja) 303 petsc = Petsc(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_name=pkg_name, pkg_arch=pkg_arch, pkg_pkgs=pkg_pkgs, verbose=verbose) 304 writer[output](petsc) 305 petsc.summary() 306 307if __name__ == '__main__': 308 import optparse 309 parser = optparse.OptionParser() 310 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 311 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 312 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 313 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 314 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 315 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 316 parser.add_option('--output', help='Location to write output file', default=None) 317 opts, extra_args = parser.parse_args() 318 if extra_args: 319 import sys 320 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 321 exit(1) 322 main(petsc_arch=opts.petsc_arch, pkg_dir=opts.pkg_dir, pkg_name=opts.pkg_name, pkg_arch=opts.pkg_arch, pkg_pkgs=opts.pkg_pkgs, output=opts.output, verbose=opts.verbose) 323