1from __future__ import print_function 2from __future__ import absolute_import 3import sys 4if not hasattr(sys, 'version_info'): 5 print('*** Python version 1 is not supported. Please get the latest version from www.python.org ***') 6 sys.exit(4) 7 8import pickle 9 10import subprocess 11 12import nargs 13 14# Uses threads to monitor running programs and time them out if they take too long 15useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 16if useThreads == 'no' or useThreads == '0': 17 useThreads = 0 18elif useThreads == None or useThreads == 'yes' or useThreads == '1': 19 useThreads = 1 20else: 21 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 22 23useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 24if useSelect == 'no' or useSelect == '0': 25 useSelect = 0 26elif useSelect is None or useSelect == 'yes' or useSelect == '1': 27 useSelect = 1 28else: 29 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 30 31# Run parts of configure in parallel, does not currently work; 32# see config/BuildSystem/config/framework.parallelQueueEvaluation() 33useParallel = nargs.Arg.findArgument('useParallel', sys.argv[1:]) 34if useParallel == 'no' or useParallel == '0': 35 useParallel = 0 36elif useParallel is None or useParallel == 'yes': 37 useParallel = 5 38else: 39 if useParallel == '1': 40 # handle case with --useParallel was used 41 found = 0 42 for i in sys.argv[1:]: 43 if i.startswith('--useParallel='): 44 found = 1 45 break 46 if found: useParallel = int(useParallel) 47 else: useParallel = 5 48useParallel = 0 49 50import logger 51 52class Script(logger.Logger): 53 def __init__(self, clArgs = None, argDB = None, log = None): 54 self.checkPython() 55 logger.Logger.__init__(self, clArgs, argDB, log) 56 self.shell = '/bin/sh' 57 self.showHelp = 1 58 return 59 60 def hasHelpFlag(self): 61 '''Decide whether to display the help message and exit''' 62 import nargs 63 64 if not self.showHelp: 65 return 0 66 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 67 return 0 68 return 1 69 70 def hasListFlag(self): 71 '''Decide whether to display the list of download files and exit''' 72 import nargs 73 74 if not self.showHelp: 75 return 0 76 if nargs.Arg.findArgument('with-packages-download-dir', self.clArgs) is None: 77 return 0 78 return 1 79 80 def setupArguments(self, argDB): 81 '''This method now also creates the help and action logs''' 82 import help 83 84 argDB = logger.Logger.setupArguments(self, argDB) 85 86 self.help = help.Help(argDB) 87 self.help.title = 'Script Help' 88 89 self.actions = help.Info(argDB) 90 self.actions.title = 'Script Actions' 91 92 self.setupHelp(self.help) 93 return argDB 94 95 def setupHelp(self, help): 96 '''This method should be overridden to provide help for arguments''' 97 import nargs 98 99 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 100 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 101 help.addArgument('Script', '-with-packages-download-dir=<dir>', nargs.ArgDir(None,None, 'Skip network download of package tarballs and locate them in specified dir. If not found in dir, print package URL - so it can be obtained manually.', isTemporary = 1), ignoreDuplicates = 1) 102 return help 103 104 def setup(self): 105 ''' This method checks to see whether help was requested''' 106 if hasattr(self, '_setup'): 107 return 108 logger.Logger.setup(self) 109 self._setup = 1 110 if self.hasHelpFlag(): 111 self.argDB.readonly = True 112 if self.argDB.target == ['default']: 113 sections = None 114 else: 115 sections = self.argDB.target 116 self.help.output(sections = sections) 117 sys.exit() 118 if self.hasListFlag(): 119 self.help.outputDownload() 120 return 121 122 def cleanup(self): 123 '''This method outputs the action log''' 124 self.actions.output(self.log) 125 return 126 127 def checkPython(self): 128 if not hasattr(sys, 'version_info') or sys.version_info < (3,4): 129 raise RuntimeError('BuildSystem requires Python version 3.4 or higher. Get Python at https://www.python.org/') 130 return 131 132 @staticmethod 133 def getModule(root, name): 134 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 135 import imp 136 137 (fp, pathname, description) = imp.find_module(name, [root]) 138 try: 139 return imp.load_module(name, fp, pathname, description) 140 finally: 141 if fp: fp.close() 142 143 @staticmethod 144 def importModule(moduleName): 145 '''Import the named module, and return the module object 146 - Works properly for fully qualified names''' 147 module = __import__(moduleName) 148 components = moduleName.split('.') 149 for comp in components[1:]: 150 module = getattr(module, comp) 151 return module 152 153 @staticmethod 154 def runShellCommand(command, log=None, cwd=None, env=None): 155 return Script.runShellCommandSeq([command], log=log, cwd=cwd, env=env) 156 157 @staticmethod 158 def runShellCommandSeq(commandseq, log=None, cwd=None, env=None): 159 Popen = subprocess.Popen 160 PIPE = subprocess.PIPE 161 output = '' 162 error = '' 163 ret = 0 164 for command in commandseq: 165 useShell = isinstance(command, str) or isinstance(command, bytes) 166 if log: log.write('Executing: %s\n' % (command,)) 167 try: 168 pipe = Popen(command, cwd=cwd, env=env, stdin=None, stdout=PIPE, stderr=PIPE, 169 shell=useShell) 170 (out, err) = pipe.communicate() 171 out = out.decode(encoding='UTF-8',errors='replace') 172 err = err.decode(encoding='UTF-8',errors='replace') 173 ret = pipe.returncode 174 except Exception as e: 175 if hasattr(e,'message') and hasattr(e,'errno'): 176 return ('', e.message, e.errno) 177 else: 178 return ('', str(e),1) 179 output += out 180 error += err 181 if ret: 182 break 183 return (output, error, ret) 184 185 @staticmethod 186 def defaultCheckCommand(command, status, output, error): 187 '''Raise an error if the exit status is nonzero 188 Since output and error may be huge and the exception error message may be printed to the 189 screen we cannot print the entire output''' 190 if status: 191 mlen = 512//2 192 if len(output) > 2*mlen: 193 output = output[0:mlen]+'\n .... more output .....\n'+output[len(output)- mlen:] 194 if len(error) > 2*mlen: 195 error = error[0:mlen]+'\n .... more error .....\n'+error[len(error)- mlen:] 196 raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 197 198 @staticmethod 199 def passCheckCommand(command, status, output, error): 200 '''Does not check the command results''' 201 202 @staticmethod 203 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 204 '''Execute a shell command returning the output, and optionally provide a custom error checker 205 - This returns a tuple of the (output, error, statuscode)''' 206 '''The timeout is ignored unless the threads values is nonzero''' 207 return Script.executeShellCommandSeq([command], checkCommand=checkCommand, timeout=timeout, log=log, lineLimit=lineLimit, cwd=cwd, env=env, logOutputflg = logOutputflg, threads = threads) 208 209 @staticmethod 210 def executeShellCommandSeq(commandseq, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 211 '''Execute a sequence of shell commands (an && chain) returning the output, and optionally provide a custom error checker 212 - This returns a tuple of the (output, error, statuscode)''' 213 if not checkCommand: 214 checkCommand = Script.defaultCheckCommand 215 if log is None: 216 log = logger.Logger.defaultLog 217 def logOutput(log, output, logOutputflg): 218 import re 219 if not logOutputflg: return output 220 # get rid of multiple blank lines 221 output = re.sub('\n+','\n', output).strip() 222 if output: 223 if lineLimit: 224 output = '\n'.join(output.split('\n')[:lineLimit]) 225 if '\n' in output: # multi-line output 226 log.write('stdout:\n'+output+'\n') 227 else: 228 log.write('stdout: '+output+'\n') 229 return output 230 def runInShell(commandseq, log, cwd, env): 231 if useThreads and threads: 232 import threading 233 log.write('Running Executable with threads to time it out at '+str(timeout)+'\n') 234 class InShell(threading.Thread): 235 def __init__(self): 236 threading.Thread.__init__(self) 237 self.name = 'Shell Command' 238 self.setDaemon(1) 239 def run(self): 240 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 241 (self.output, self.error, self.status) = Script.runShellCommandSeq(commandseq, log, cwd, env) 242 thread = InShell() 243 thread.start() 244 thread.join(timeout) 245 if thread.is_alive(): 246 error = 'Runaway process exceeded time limit of '+str(timeout)+'\n' 247 log.write(error) 248 return ('', error, -1) 249 else: 250 return (thread.output, thread.error, thread.status) 251 else: 252 return Script.runShellCommandSeq(commandseq, log, cwd, env) 253 254 (output, error, status) = runInShell(commandseq, log, cwd, env) 255 output = logOutput(log, output,logOutputflg) 256 checkCommand(commandseq, status, output, error) 257 return (output, error, status) 258 259 def loadConfigure(self, argDB = None): 260 if argDB is None: 261 argDB = self.argDB 262 if not 'configureCache' in argDB: 263 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 264 return None 265 try: 266 cache = argDB['configureCache'] 267 framework = pickle.loads(cache) 268 framework.framework = framework 269 framework.argDB = argDB 270 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 271 except pickle.UnpicklingError as e: 272 framework = None 273 self.logPrint('Invalid cached configure: '+str(e)) 274 return framework 275 276import args 277 278class LanguageProcessor(args.ArgumentProcessor): 279 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 280 self.languageModule = {} 281 self.preprocessorObject = {} 282 self.compilerObject = {} 283 self.linkerObject = {} 284 self.sharedLinkerObject = {} 285 self.dynamicLinkerObject = {} 286 self.framework = framework 287 self.versionControl = versionControl 288 args.ArgumentProcessor.__init__(self, clArgs, argDB) 289 self.outputFiles = {} 290 self.modulePath = 'config.compile' 291 return 292 293 def getCompilers(self): 294 if self.framework is None: 295 return 296 return self.framework.require('config.compilers', None) 297 compilers = property(getCompilers, doc = 'The config.compilers configure object') 298 def getLibraries(self): 299 if self.framework is None: 300 return 301 return self.framework.require('config.libraries', None) 302 libraries = property(getLibraries, doc = 'The config.libraries configure object') 303 304 def __getstate__(self, d = None): 305 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 306 if d is None: 307 d = args.ArgumentProcessor.__getstate__(self) 308 if 'languageModule' in d: 309 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 310 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 311 if member in d: 312 del d[member] 313 return d 314 315 def __setstate__(self, d): 316 '''We must create the language modules''' 317 args.ArgumentProcessor.__setstate__(self, d) 318 self.__dict__.update(d) 319 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 320 self.preprocessorObject = {} 321 self.compilerObject = {} 322 self.linkerObject = {} 323 self.sharedLinkerObject = {} 324 self.dynamicLinkerObject = {} 325 return 326 327 def setArgDB(self, argDB): 328 args.ArgumentProcessor.setArgDB(self, argDB) 329 for obj in self.preprocessorObject.values(): 330 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 331 obj.argDB = argDB 332 for obj in self.compilerObject.values(): 333 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 334 obj.argDB = argDB 335 for obj in self.linkerObject.values(): 336 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 337 obj.argDB = argDB 338 for obj in self.sharedLinkerObject.values(): 339 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 340 obj.argDB = argDB 341 for obj in self.dynamicLinkerObject.values(): 342 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 343 obj.argDB = argDB 344 if not self.compilers is None: 345 self.compilers.argDB = argDB 346 for obj in self.preprocessorObject.values(): 347 if hasattr(obj, 'configCompilers'): 348 obj.configCompilers.argDB = argDB 349 for obj in self.compilerObject.values(): 350 if hasattr(obj, 'configCompilers'): 351 obj.configCompilers.argDB = argDB 352 for obj in self.linkerObject.values(): 353 if hasattr(obj, 'configCompilers'): 354 obj.configCompilers.argDB = argDB 355 for obj in self.sharedLinkerObject.values(): 356 if hasattr(obj, 'configCompilers'): 357 obj.configCompilers.argDB = argDB 358 for obj in self.dynamicLinkerObject.values(): 359 if hasattr(obj, 'configCompilers'): 360 obj.configCompilers.argDB = argDB 361 if not self.libraries is None: 362 self.libraries.argDB = argDB 363 for obj in self.linkerObject.values(): 364 if hasattr(obj, 'configLibraries'): 365 obj.configLibraries.argDB = argDB 366 for obj in self.sharedLinkerObject.values(): 367 if hasattr(obj, 'configLibraries'): 368 obj.configLibraries.argDB = argDB 369 for obj in self.dynamicLinkerObject.values(): 370 if hasattr(obj, 'configLibraries'): 371 obj.configLibraries.argDB = argDB 372 return 373 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 374 375 def getLanguageModule(self, language, moduleName = None): 376 '''Return the module associated with operations for a given language 377 - Giving a moduleName explicitly forces a reimport''' 378 if not language in self.languageModule or not moduleName is None: 379 try: 380 if moduleName is None: 381 moduleName = self.modulePath+'.'+language 382 module = __import__(moduleName) 383 except ImportError as e: 384 if not moduleName is None: 385 self.logPrint('Failure to find language module: '+str(e)) 386 try: 387 moduleName = self.modulePath+'.'+language 388 module = __import__(moduleName) 389 except ImportError as e: 390 self.logPrint('Failure to find language module: '+str(e)) 391 moduleName = 'config.compile.'+language 392 module = __import__(moduleName) 393 components = moduleName.split('.') 394 for component in components[1:]: 395 module = getattr(module, component) 396 module._loadName = moduleName 397 self.languageModule[language] = module 398 return self.languageModule[language] 399 400 def getPreprocessorObject(self, language): 401 if not language in self.preprocessorObject: 402 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 403 self.preprocessorObject[language].setup() 404 if not self.compilers is None: 405 self.preprocessorObject[language].configCompilers = self.compilers 406 if not self.versionControl is None: 407 self.preprocessorObject[language].versionControl = self.versionControl 408 return self.preprocessorObject[language] 409 410 def setPreprocessorObject(self, language, preprocessor): 411 self.preprocessorObject[language] = preprocessor 412 return self.getPreprocessorObject(language) 413 414 def getCompilerObject(self, language): 415 if not language in self.compilerObject: 416 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 417 self.compilerObject[language].setup() 418 if not self.compilers is None: 419 self.compilerObject[language].configCompilers = self.compilers 420 if not self.versionControl is None: 421 self.compilerObject[language].versionControl = self.versionControl 422 return self.compilerObject[language] 423 424 def setCompilerObject(self, language, compiler): 425 self.compilerObject[language] = compiler 426 return self.getCompilerObject(language) 427 428 def getLinkerObject(self, language): 429 if not language in self.linkerObject: 430 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 431 self.linkerObject[language].setup() 432 if not self.compilers is None: 433 self.linkerObject[language].configCompilers = self.compilers 434 if not self.libraries is None: 435 self.linkerObject[language].configLibraries = self.libraries 436 if not self.versionControl is None: 437 self.linkerObject[language].versionControl = self.versionControl 438 return self.linkerObject[language] 439 440 def setLinkerObject(self, language, linker): 441 self.linkerObject[language] = linker 442 return self.getLinkerObject(language) 443 444 def getSharedLinkerObject(self, language): 445 if not language in self.sharedLinkerObject: 446 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 447 self.sharedLinkerObject[language].setup() 448 if not self.compilers is None: 449 self.sharedLinkerObject[language].configCompilers = self.compilers 450 if not self.libraries is None: 451 self.sharedLinkerObject[language].configLibraries = self.libraries 452 if not self.versionControl is None: 453 self.sharedLinkerObject[language].versionControl = self.versionControl 454 return self.sharedLinkerObject[language] 455 456 def setSharedLinkerObject(self, language, linker): 457 self.sharedLinkerObject[language] = linker 458 return self.getSharedLinkerObject(language) 459 460 def getDynamicLinkerObject(self, language): 461 if not language in self.dynamicLinkerObject: 462 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 463 self.dynamicLinkerObject[language].setup() 464 if not self.compilers is None: 465 self.dynamicLinkerObject[language].configCompilers = self.compilers 466 if not self.libraries is None: 467 self.dynamicLinkerObject[language].configLibraries = self.libraries 468 if not self.versionControl is None: 469 self.dynamicLinkerObject[language].versionControl = self.versionControl 470 return self.dynamicLinkerObject[language] 471 472 def setDynamicLinkerObject(self, language, linker): 473 self.dynamicLinkerObject[language] = linker 474 return self.getDynamicLinkerObject(language) 475