1from __future__ import print_function 2from __future__ import absolute_import 3import sys 4if not hasattr(sys, 'version_info'): 5 print('*** Python version 1 is not supported. Please get the latest version from www.python.org ***') 6 sys.exit(4) 7 8import pickle 9 10import subprocess 11 12import nargs 13 14# Uses threads to monitor running programs and time them out if they take too long 15useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 16if useThreads == 'no' or useThreads == '0': 17 useThreads = 0 18elif useThreads == None or useThreads == 'yes' or useThreads == '1': 19 useThreads = 1 20else: 21 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 22 23useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 24if useSelect == 'no' or useSelect == '0': 25 useSelect = 0 26elif useSelect is None or useSelect == 'yes' or useSelect == '1': 27 useSelect = 1 28else: 29 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 30 31# Run parts of configure in parallel, does not currently work; 32# see config/BuildSystem/config/framework.parallelQueueEvaluation() 33useParallel = nargs.Arg.findArgument('useParallel', sys.argv[1:]) 34if useParallel == 'no' or useParallel == '0': 35 useParallel = 0 36elif useParallel is None or useParallel == 'yes': 37 useParallel = 5 38else: 39 if useParallel == '1': 40 # handle case with --useParallel was used 41 found = 0 42 for i in sys.argv[1:]: 43 if i.startswith('--useParallel='): 44 found = 1 45 break 46 if found: useParallel = int(useParallel) 47 else: useParallel = 5 48useParallel = 0 49 50import logger 51 52class Script(logger.Logger): 53 def __init__(self, clArgs = None, argDB = None, log = None): 54 self.checkPython() 55 logger.Logger.__init__(self, clArgs, argDB, log) 56 self.shell = '/bin/sh' 57 self.showHelp = 1 58 return 59 60 def hasHelpFlag(self): 61 '''Decide whether to display the help message and exit''' 62 import nargs 63 64 if not self.showHelp: 65 return 0 66 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 67 return 0 68 return 1 69 70 def hasListFlag(self): 71 '''Decide whether to display the list of download files and exit''' 72 import nargs 73 74 if not self.showHelp: 75 return 0 76 if nargs.Arg.findArgument('with-packages-download-dir', self.clArgs) is None: 77 return 0 78 return 1 79 80 def setupArguments(self, argDB): 81 '''This method now also creates the help and action logs''' 82 import help 83 84 argDB = logger.Logger.setupArguments(self, argDB) 85 86 self.help = help.Help(argDB) 87 self.help.title = 'Script Help' 88 89 self.actions = help.Info(argDB) 90 self.actions.title = 'Script Actions' 91 92 self.setupHelp(self.help) 93 return argDB 94 95 def setupHelp(self, help): 96 '''This method should be overridden to provide help for arguments''' 97 import nargs 98 99 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 100 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 101 help.addArgument('Script', '-with-packages-download-dir=<dir>', nargs.ArgDir(None,None, 'Skip network download of package tarballs and locate them in specified dir. If not found in dir, print package URL - so it can be obtained manually.', isTemporary = 1), ignoreDuplicates = 1) 102 return help 103 104 def setup(self): 105 ''' This method checks to see whether help was requested''' 106 if hasattr(self, '_setup'): 107 return 108 logger.Logger.setup(self) 109 self._setup = 1 110 if self.hasHelpFlag(): 111 self.argDB.readonly = True 112 if self.argDB.target == ['default']: 113 sections = None 114 else: 115 sections = self.argDB.target 116 self.help.output(sections = sections) 117 sys.exit() 118 if self.hasListFlag(): 119 self.help.outputDownload() 120 return 121 122 def cleanup(self): 123 '''This method outputs the action log''' 124 self.actions.output(self.log) 125 return 126 127 def checkPython(self): 128 if not hasattr(sys, 'version_info') or sys.version_info < (3,4): 129 raise RuntimeError('BuildSystem requires Python version 3.4 or higher. Get Python at https://www.python.org/') 130 return 131 132 @staticmethod 133 def getModule(root, name): 134 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 135 if sys.version_info < (3,12): 136 import imp 137 (fp, pathname, description) = imp.find_module(name, [root]) 138 try: 139 return imp.load_module(name, fp, pathname, description) 140 finally: 141 if fp: fp.close() 142 else: 143 import importlib.util 144 spec = importlib.util.spec_from_file_location(name, root) 145 module = importlib.util.module_from_spec(spec) # novermin 146 sys.modules[name] = module 147 spec.loader.exec_module(module) 148 149 @staticmethod 150 def importModule(moduleName): 151 '''Import the named module, and return the module object 152 - Works properly for fully qualified names''' 153 module = __import__(moduleName) 154 components = moduleName.split('.') 155 for comp in components[1:]: 156 module = getattr(module, comp) 157 return module 158 159 @staticmethod 160 def runShellCommand(command, log=None, cwd=None, env=None): 161 return Script.runShellCommandSeq([command], log=log, cwd=cwd, env=env) 162 163 @staticmethod 164 def runShellCommandSeq(commandseq, log=None, cwd=None, env=None): 165 Popen = subprocess.Popen 166 PIPE = subprocess.PIPE 167 output = '' 168 error = '' 169 ret = 0 170 for command in commandseq: 171 useShell = isinstance(command, str) or isinstance(command, bytes) 172 if log: log.write('Executing: %s\n' % (command,)) 173 try: 174 pipe = Popen(command, cwd=cwd, env=env, stdin=None, stdout=PIPE, stderr=PIPE, 175 shell=useShell) 176 (out, err) = pipe.communicate() 177 out = out.decode(encoding='UTF-8',errors='replace') 178 err = err.decode(encoding='UTF-8',errors='replace') 179 ret = pipe.returncode 180 except Exception as e: 181 if hasattr(e,'message') and hasattr(e,'errno'): 182 return ('', e.message, e.errno) 183 else: 184 return ('', str(e),1) 185 output += out 186 error += err 187 if ret: 188 break 189 return (output, error, ret) 190 191 @staticmethod 192 def defaultCheckCommand(command, status, output, error): 193 '''Raise an error if the exit status is nonzero''' 194 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 195 196 @staticmethod 197 def passCheckCommand(command, status, output, error): 198 '''Does not check the command results''' 199 200 @staticmethod 201 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 202 '''Execute a shell command returning the output, and optionally provide a custom error checker 203 - This returns a tuple of the (output, error, statuscode)''' 204 '''The timeout is ignored unless the threads values is nonzero''' 205 return Script.executeShellCommandSeq([command], checkCommand=checkCommand, timeout=timeout, log=log, lineLimit=lineLimit, cwd=cwd, env=env, logOutputflg = logOutputflg, threads = threads) 206 207 @staticmethod 208 def executeShellCommandSeq(commandseq, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 209 '''Execute a sequence of shell commands (an && chain) returning the output, and optionally provide a custom error checker 210 - This returns a tuple of the (output, error, statuscode)''' 211 if not checkCommand: 212 checkCommand = Script.defaultCheckCommand 213 if log is None: 214 log = logger.Logger.defaultLog 215 def logOutput(log, output, logOutputflg): 216 import re 217 if not logOutputflg: return output 218 # get rid of multiple blank lines 219 output = re.sub('\n+','\n', output).strip() 220 if output: 221 if lineLimit: 222 output = '\n'.join(output.split('\n')[:lineLimit]) 223 if '\n' in output: # multi-line output 224 log.write('stdout:\n'+output+'\n') 225 else: 226 log.write('stdout: '+output+'\n') 227 return output 228 def runInShell(commandseq, log, cwd, env): 229 if useThreads and threads: 230 import threading 231 log.write('Running Executable with threads to time it out at '+str(timeout)+'\n') 232 class InShell(threading.Thread): 233 def __init__(self): 234 threading.Thread.__init__(self) 235 self.name = 'Shell Command' 236 self.setDaemon(1) 237 def run(self): 238 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 239 (self.output, self.error, self.status) = Script.runShellCommandSeq(commandseq, log, cwd, env) 240 thread = InShell() 241 thread.start() 242 thread.join(timeout) 243 if thread.is_alive(): 244 error = 'Runaway process exceeded time limit of '+str(timeout)+'\n' 245 log.write(error) 246 return ('', error, -1) 247 else: 248 return (thread.output, thread.error, thread.status) 249 else: 250 return Script.runShellCommandSeq(commandseq, log, cwd, env) 251 252 (output, error, status) = runInShell(commandseq, log, cwd, env) 253 output = logOutput(log, output,logOutputflg) 254 checkCommand(commandseq, status, output, error) 255 return (output, error, status) 256 257 def loadConfigure(self, argDB = None): 258 if argDB is None: 259 argDB = self.argDB 260 if not 'configureCache' in argDB: 261 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 262 return None 263 try: 264 cache = argDB['configureCache'] 265 framework = pickle.loads(cache) 266 framework.framework = framework 267 framework.argDB = argDB 268 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 269 except pickle.UnpicklingError as e: 270 framework = None 271 self.logPrint('Invalid cached configure: '+str(e)) 272 return framework 273 274import args 275 276class LanguageProcessor(args.ArgumentProcessor): 277 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 278 self.languageModule = {} 279 self.preprocessorObject = {} 280 self.compilerObject = {} 281 self.linkerObject = {} 282 self.sharedLinkerObject = {} 283 self.dynamicLinkerObject = {} 284 self.framework = framework 285 self.versionControl = versionControl 286 args.ArgumentProcessor.__init__(self, clArgs, argDB) 287 self.outputFiles = {} 288 self.modulePath = 'config.compile' 289 return 290 291 def getCompilers(self): 292 if self.framework is None: 293 return 294 return self.framework.require('config.compilers', None) 295 compilers = property(getCompilers, doc = 'The config.compilers configure object') 296 def getLibraries(self): 297 if self.framework is None: 298 return 299 return self.framework.require('config.libraries', None) 300 libraries = property(getLibraries, doc = 'The config.libraries configure object') 301 302 def __getstate__(self, d = None): 303 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 304 if d is None: 305 d = args.ArgumentProcessor.__getstate__(self) 306 if 'languageModule' in d: 307 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 308 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 309 if member in d: 310 del d[member] 311 return d 312 313 def __setstate__(self, d): 314 '''We must create the language modules''' 315 args.ArgumentProcessor.__setstate__(self, d) 316 self.__dict__.update(d) 317 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 318 self.preprocessorObject = {} 319 self.compilerObject = {} 320 self.linkerObject = {} 321 self.sharedLinkerObject = {} 322 self.dynamicLinkerObject = {} 323 return 324 325 def setArgDB(self, argDB): 326 args.ArgumentProcessor.setArgDB(self, argDB) 327 for obj in self.preprocessorObject.values(): 328 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 329 obj.argDB = argDB 330 for obj in self.compilerObject.values(): 331 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 332 obj.argDB = argDB 333 for obj in self.linkerObject.values(): 334 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 335 obj.argDB = argDB 336 for obj in self.sharedLinkerObject.values(): 337 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 338 obj.argDB = argDB 339 for obj in self.dynamicLinkerObject.values(): 340 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 341 obj.argDB = argDB 342 if not self.compilers is None: 343 self.compilers.argDB = argDB 344 for obj in self.preprocessorObject.values(): 345 if hasattr(obj, 'configCompilers'): 346 obj.configCompilers.argDB = argDB 347 for obj in self.compilerObject.values(): 348 if hasattr(obj, 'configCompilers'): 349 obj.configCompilers.argDB = argDB 350 for obj in self.linkerObject.values(): 351 if hasattr(obj, 'configCompilers'): 352 obj.configCompilers.argDB = argDB 353 for obj in self.sharedLinkerObject.values(): 354 if hasattr(obj, 'configCompilers'): 355 obj.configCompilers.argDB = argDB 356 for obj in self.dynamicLinkerObject.values(): 357 if hasattr(obj, 'configCompilers'): 358 obj.configCompilers.argDB = argDB 359 if not self.libraries is None: 360 self.libraries.argDB = argDB 361 for obj in self.linkerObject.values(): 362 if hasattr(obj, 'configLibraries'): 363 obj.configLibraries.argDB = argDB 364 for obj in self.sharedLinkerObject.values(): 365 if hasattr(obj, 'configLibraries'): 366 obj.configLibraries.argDB = argDB 367 for obj in self.dynamicLinkerObject.values(): 368 if hasattr(obj, 'configLibraries'): 369 obj.configLibraries.argDB = argDB 370 return 371 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 372 373 def getLanguageModule(self, language, moduleName = None): 374 '''Return the module associated with operations for a given language 375 - Giving a moduleName explicitly forces a reimport''' 376 if not language in self.languageModule or not moduleName is None: 377 try: 378 if moduleName is None: 379 moduleName = self.modulePath+'.'+language 380 module = __import__(moduleName) 381 except ImportError as e: 382 if not moduleName is None: 383 self.logPrint('Failure to find language module: '+str(e)) 384 try: 385 moduleName = self.modulePath+'.'+language 386 module = __import__(moduleName) 387 except ImportError as e: 388 self.logPrint('Failure to find language module: '+str(e)) 389 moduleName = 'config.compile.'+language 390 module = __import__(moduleName) 391 components = moduleName.split('.') 392 for component in components[1:]: 393 module = getattr(module, component) 394 module._loadName = moduleName 395 self.languageModule[language] = module 396 return self.languageModule[language] 397 398 def getPreprocessorObject(self, language): 399 if not language in self.preprocessorObject: 400 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 401 self.preprocessorObject[language].setup() 402 if not self.compilers is None: 403 self.preprocessorObject[language].configCompilers = self.compilers 404 if not self.versionControl is None: 405 self.preprocessorObject[language].versionControl = self.versionControl 406 return self.preprocessorObject[language] 407 408 def setPreprocessorObject(self, language, preprocessor): 409 self.preprocessorObject[language] = preprocessor 410 return self.getPreprocessorObject(language) 411 412 def getCompilerObject(self, language): 413 if not language in self.compilerObject: 414 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 415 self.compilerObject[language].setup() 416 if not self.compilers is None: 417 self.compilerObject[language].configCompilers = self.compilers 418 if not self.versionControl is None: 419 self.compilerObject[language].versionControl = self.versionControl 420 return self.compilerObject[language] 421 422 def setCompilerObject(self, language, compiler): 423 self.compilerObject[language] = compiler 424 return self.getCompilerObject(language) 425 426 def getLinkerObject(self, language): 427 if not language in self.linkerObject: 428 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 429 self.linkerObject[language].setup() 430 if not self.compilers is None: 431 self.linkerObject[language].configCompilers = self.compilers 432 if not self.libraries is None: 433 self.linkerObject[language].configLibraries = self.libraries 434 if not self.versionControl is None: 435 self.linkerObject[language].versionControl = self.versionControl 436 return self.linkerObject[language] 437 438 def setLinkerObject(self, language, linker): 439 self.linkerObject[language] = linker 440 return self.getLinkerObject(language) 441 442 def getSharedLinkerObject(self, language): 443 if not language in self.sharedLinkerObject: 444 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 445 self.sharedLinkerObject[language].setup() 446 if not self.compilers is None: 447 self.sharedLinkerObject[language].configCompilers = self.compilers 448 if not self.libraries is None: 449 self.sharedLinkerObject[language].configLibraries = self.libraries 450 if not self.versionControl is None: 451 self.sharedLinkerObject[language].versionControl = self.versionControl 452 return self.sharedLinkerObject[language] 453 454 def setSharedLinkerObject(self, language, linker): 455 self.sharedLinkerObject[language] = linker 456 return self.getSharedLinkerObject(language) 457 458 def getDynamicLinkerObject(self, language): 459 if not language in self.dynamicLinkerObject: 460 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 461 self.dynamicLinkerObject[language].setup() 462 if not self.compilers is None: 463 self.dynamicLinkerObject[language].configCompilers = self.compilers 464 if not self.libraries is None: 465 self.dynamicLinkerObject[language].configLibraries = self.libraries 466 if not self.versionControl is None: 467 self.dynamicLinkerObject[language].versionControl = self.versionControl 468 return self.dynamicLinkerObject[language] 469 470 def setDynamicLinkerObject(self, language, linker): 471 self.dynamicLinkerObject[language] = linker 472 return self.getDynamicLinkerObject(language) 473