1from __future__ import absolute_import 2import logger 3 4import os 5try: 6 from urllib import urlretrieve 7except ImportError: 8 from urllib.request import urlretrieve 9try: 10 import urlparse as urlparse_local # novermin 11except ImportError: 12 from urllib import parse as urlparse_local # novermin 13import config.base 14import socket 15import shutil 16 17# Fix parsing for nonstandard schemes 18urlparse_local.uses_netloc.extend(['bk', 'ssh', 'svn']) 19 20class Retriever(logger.Logger): 21 def __init__(self, sourceControl, clArgs = None, argDB = None): 22 logger.Logger.__init__(self, clArgs, argDB) 23 self.sourceControl = sourceControl 24 self.stamp = None 25 return 26 27 def isDirectoryGitRepo(self, directory): 28 from config.base import Configure 29 for loc in ['.git','']: 30 cmd = '%s rev-parse --resolve-git-dir %s' % (self.sourceControl.git, os.path.join(directory,loc)) 31 (output, error, ret) = Configure.executeShellCommand(cmd, checkCommand = Configure.passCheckCommand, log = self.log) 32 if not ret: 33 return True 34 return False 35 36 @staticmethod 37 def removeTarget(t): 38 if os.path.islink(t) or os.path.isfile(t): 39 os.unlink(t) # same as os.remove(t) 40 elif os.path.isdir(t): 41 shutil.rmtree(t) 42 43 @staticmethod 44 def getDownloadFailureMessage(package, url, filename=None): 45 slashFilename = '/'+filename if filename else '' 46 return '''\ 47Unable to download package %s from: %s 48* If URL specified manually - perhaps there is a typo? 49* If your network is disconnected - please reconnect and rerun ./configure 50* Or perhaps you have a firewall blocking the download 51* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 52* or you can download the above URL manually, to /yourselectedlocation%s 53 and use the configure option: 54 --download-%s=/yourselectedlocation%s 55 ''' % (package.upper(), url, slashFilename, package, slashFilename) 56 57 @staticmethod 58 def removePrefix(url,prefix): 59 '''Replacement for str.removeprefix() supported only since Python 3.9''' 60 if url.startswith(prefix): 61 return url[len(prefix):] 62 return url 63 64 def genericRetrieve(self, url, root, package, submodules): 65 '''Fetch package from version control repository or tarfile indicated by URL and extract it into root''' 66 67 parsed = urlparse_local.urlparse(url) 68 if parsed[0] == 'dir': 69 f = self.dirRetrieve 70 elif parsed[0] == 'link': 71 f = self.linkRetrieve 72 elif parsed[0] == 'git': 73 f = self.gitRetrieve 74 elif parsed[0] == 'ssh' and parsed[2].endswith('.git'): 75 f = self.gitRetrieve 76 elif parsed[0] == 'https' and parsed[2].endswith('.git'): 77 f = self.gitRetrieve 78 elif parsed[0] == 'hg': 79 f = self.hgRetrieve 80 elif parsed[0] == 'ssh' and parsed[1].startswith('hg@'): 81 f = self.hgRetrieve 82 elif os.path.isdir(url): 83 if self.isDirectoryGitRepo(url): 84 f = self.gitRetrieve 85 else: 86 f = self.dirRetrieve 87 else: 88 f = self.tarballRetrieve 89 return f(url, root, package, submodules) 90 91 def dirRetrieve(self, url, root, package, submodules): 92 self.logPrint('Retrieving %s as directory' % url, 3, 'install') 93 d = self.removePrefix(url, 'dir://') 94 if not os.path.isdir(d): raise RuntimeError('URL %s is not a directory' % url) 95 96 t = os.path.join(root,os.path.basename(d)) 97 self.removeTarget(t) 98 shutil.copytree(d,t) 99 100 def linkRetrieve(self, url, root, package, submodules): 101 self.logPrint('Retrieving %s as link' % url, 3, 'install') 102 d = self.removePrefix(url, 'link://') 103 if not os.path.isdir(d): raise RuntimeError('URL %s is not pointing to a directory' % url) 104 105 t = os.path.join(root,os.path.basename(d)) 106 self.removeTarget(t) 107 os.symlink(os.path.abspath(d),t) 108 109 def gitRetrieve(self, url, root, package, submodules): 110 self.logPrint('Retrieving %s as git repo' % url, 3, 'install') 111 if not hasattr(self.sourceControl, 'git'): 112 raise RuntimeError('self.sourceControl.git not set') 113 d = self.removePrefix(url, 'git://') 114 if os.path.isdir(d) and not self.isDirectoryGitRepo(d): 115 raise RuntimeError('URL %s is a directory but not a git repository' % url) 116 117 newgitrepo = os.path.join(root,'git.'+package) 118 self.removeTarget(newgitrepo) 119 120 try: 121 submodopt ='' 122 for itm in submodules: 123 submodopt += ' --recurse-submodules='+itm 124 config.base.Configure.executeShellCommand('%s clone %s %s %s' % (self.sourceControl.git, submodopt, d, newgitrepo), log = self.log, timeout = 120.0) 125 except RuntimeError as e: 126 self.logPrint('ERROR: '+str(e)) 127 err = str(e) 128 failureMessage = self.getDownloadFailureMessage(package, url) 129 raise RuntimeError('Unable to clone '+package+'\n'+err+failureMessage) 130 131 def hgRetrieve(self, url, root, package, submodules): 132 self.logPrint('Retrieving %s as hg repo' % url, 3, 'install') 133 if not hasattr(self.sourceControl, 'hg'): 134 raise RuntimeError('self.sourceControl.hg not set') 135 d = self.removePrefix(url, 'hg://') 136 137 newgitrepo = os.path.join(root,'hg.'+package) 138 self.removeTarget(newgitrepo) 139 try: 140 config.base.Configure.executeShellCommand('%s clone %s %s' % (self.sourceControl.hg, d, newgitrepo), log = self.log, timeout = 120.0) 141 except RuntimeError as e: 142 self.logPrint('ERROR: '+str(e)) 143 err = str(e) 144 failureMessage = self.getDownloadFailureMessage(package, url) 145 raise RuntimeError('Unable to clone '+package+'\n'+err+failureMessage) 146 147 def tarballRetrieve(self, url, root, package, submodules): 148 parsed = urlparse_local.urlparse(url) 149 filename = os.path.basename(parsed[2]) 150 localFile = os.path.join(root,'_d_'+filename) 151 self.logPrint('Retrieving %s as tarball to %s' % (url,localFile) , 3, 'install') 152 ext = os.path.splitext(localFile)[1] 153 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 154 raise RuntimeError('Unknown compression type in URL: '+ url) 155 156 self.removeTarget(localFile) 157 158 if parsed[0] == 'file' and not parsed[1]: 159 url = parsed[2] 160 if os.path.exists(url): 161 if not os.path.isfile(url): 162 raise RuntimeError('Local path exists but is not a regular file: '+ url) 163 # copy local file 164 shutil.copyfile(url, localFile) 165 else: 166 # fetch remote file 167 try: 168 sav_timeout = socket.getdefaulttimeout() 169 socket.setdefaulttimeout(30) 170 urlretrieve(url, localFile) 171 socket.setdefaulttimeout(sav_timeout) 172 except Exception as e: 173 socket.setdefaulttimeout(sav_timeout) 174 failureMessage = self.getDownloadFailureMessage(package, url, filename) 175 raise RuntimeError(failureMessage) 176 177 self.logPrint('Extracting '+localFile) 178 if ext in ['.zip','.ZIP']: 179 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 180 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 181 dirname = os.path.normpath(output[0].strip()) 182 else: 183 failureMessage = '''\ 184Downloaded package %s from: %s is not a tarball. 185[or installed python cannot process compressed files] 186* If you are behind a firewall - please fix your proxy and rerun ./configure 187 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 188* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 189* or you can download the above URL manually, to /yourselectedlocation/%s 190 and use the configure option: 191 --download-%s=/yourselectedlocation/%s 192''' % (package.upper(), url, filename, package, filename) 193 import tarfile 194 try: 195 tf = tarfile.open(os.path.join(root, localFile)) 196 except tarfile.ReadError as e: 197 raise RuntimeError(str(e)+'\n'+failureMessage) 198 if not tf: raise RuntimeError(failureMessage) 199 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 200 firstname = tf.getnames()[0] 201 if firstname == 'pax_global_header': 202 firstmember = tf.getmembers()[1] 203 else: 204 firstmember = tf.getmembers()[0] 205 # some tarfiles list packagename/ but some list packagename/filename in the first entry 206 if firstmember.isdir(): 207 dirname = firstmember.name 208 else: 209 dirname = os.path.dirname(firstmember.name) 210 tf.extractall(root) 211 tf.close() 212 213 # fix file permissions for the untared tarballs. 214 try: 215 # check if 'dirname' is set' 216 if dirname: 217 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 218 else: 219 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 220 except RuntimeError as e: 221 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 222 os.unlink(localFile) 223