1from __future__ import absolute_import 2import logger 3 4import os 5try: 6 from urllib import urlretrieve 7except ImportError: 8 from urllib.request import urlretrieve 9try: 10 import urlparse as urlparse_local # novermin 11except ImportError: 12 from urllib import parse as urlparse_local 13import config.base 14import socket 15 16# Fix parsing for nonstandard schemes 17urlparse_local.uses_netloc.extend(['bk', 'ssh', 'svn']) 18 19class Retriever(logger.Logger): 20 def __init__(self, sourceControl, clArgs = None, argDB = None): 21 logger.Logger.__init__(self, clArgs, argDB) 22 self.sourceControl = sourceControl 23 self.stamp = None 24 return 25 26 def getAuthorizedUrl(self, url): 27 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 28 (scheme, location, path, parameters, query, fragment) = urlparse_local.urlparse(url) 29 if not location: 30 url = urlparse_local.urlunparse(('', '', path, parameters, query, fragment)) 31 authUrl = None 32 wasAuth = 0 33 else: 34 index = location.find('@') 35 if index >= 0: 36 login = location[0:index] 37 authUrl = url 38 url = urlparse_local.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 39 wasAuth = 1 40 else: 41 login = location.split('.')[0] 42 authUrl = urlparse_local.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 43 wasAuth = 0 44 return (url, authUrl, wasAuth) 45 46 def testAuthorizedUrl(self, authUrl): 47 '''Raise an exception if the URL cannot receive an SSH login without a password''' 48 if not authUrl: 49 raise RuntimeError('Url is empty') 50 (scheme, location, path, parameters, query, fragment) = urlparse_local.urlparse(authUrl) 51 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location, log = self.log) 52 53 def genericRetrieve(self, url, root, package): 54 '''Fetch the gzipped tarfile indicated by url and expand it into root 55 - All the logic for removing old versions, updating etc. must move''' 56 57 # copy a directory 58 if url.startswith('dir://'): 59 import shutil 60 dir = url[6:] 61 if not os.path.isdir(dir): raise RuntimeError('Url begins with dir:// but is not a directory') 62 63 if os.path.isdir(os.path.join(root,os.path.basename(dir))): shutil.rmtree(os.path.join(root,os.path.basename(dir))) 64 if os.path.isfile(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 65 66 shutil.copytree(dir,os.path.join(root,os.path.basename(dir))) 67 return 68 69 if url.startswith('link://'): 70 import shutil 71 dir = url[7:] 72 if not os.path.isdir(dir): raise RuntimeError('Url begins with link:// but it is not pointing to a directory') 73 74 if os.path.islink(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 75 if os.path.isfile(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 76 if os.path.isdir(os.path.join(root,os.path.basename(dir))): shutil.rmtree(os.path.join(root,os.path.basename(dir))) 77 os.symlink(os.path.abspath(dir),os.path.join(root,os.path.basename(dir))) 78 return 79 80 if url.startswith('git://'): 81 if not hasattr(self.sourceControl, 'git'): return 82 import shutil 83 dir = url[6:] 84 if os.path.isdir(dir): 85 if not os.path.isdir(os.path.join(dir,'.git')): raise RuntimeError('Url begins with git:// and is a directory but but does not have a .git subdirectory') 86 87 newgitrepo = os.path.join(root,'git.'+package) 88 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 89 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 90 91 try: 92 config.base.Configure.executeShellCommand(self.sourceControl.git+' clone '+dir+' '+newgitrepo, log = self.log) 93 except RuntimeError as e: 94 self.logPrint('ERROR: '+str(e)) 95 err = str(e) 96 failureMessage = '''\ 97Unable to download package %s from: %s 98* If URL specified manually - perhaps there is a typo? 99* If your network is disconnected - please reconnect and rerun ./configure 100* Or perhaps you have a firewall blocking the download 101* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 102* or you can download the above URL manually, to /yourselectedlocation 103 and use the configure option: 104 --download-%s=/yourselectedlocation 105''' % (package.upper(), url, package) 106 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 107 return 108 109 if url.startswith('hg://'): 110 if not hasattr(self.sourceControl, 'hg'): return 111 112 newgitrepo = os.path.join(root,'hg.'+package) 113 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 114 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 115 try: 116 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url[5:]+' '+newgitrepo) 117 except RuntimeError as e: 118 self.logPrint('ERROR: '+str(e)) 119 err = str(e) 120 failureMessage = '''\ 121Unable to download package %s from: %s 122* If URL specified manually - perhaps there is a typo? 123* If your network is disconnected - please reconnect and rerun ./configure 124* Or perhaps you have a firewall blocking the download 125* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 126* or you can download the above URL manually, to /yourselectedlocation 127 and use the configure option: 128 --download-%s=/yourselectedlocation 129''' % (package.upper(), url, package) 130 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 131 return 132 133 if url.startswith('ssh://hg@'): 134 if not hasattr(self.sourceControl, 'hg'): return 135 136 newgitrepo = os.path.join(root,'hg.'+package) 137 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 138 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 139 try: 140 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url+' '+newgitrepo) 141 except RuntimeError as e: 142 self.logPrint('ERROR: '+str(e)) 143 err = str(e) 144 failureMessage = '''\ 145Unable to download package %s from: %s 146* If URL specified manually - perhaps there is a typo? 147* If your network is disconnected - please reconnect and rerun ./configure 148* Or perhaps you have a firewall blocking the download 149* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 150* or you can download the above URL manually, to /yourselectedlocation 151 and use the configure option: 152 --download-%s=/yourselectedlocation 153''' % (package.upper(), url, package) 154 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 155 return 156 157 # get the tarball file name from the URL 158 filename = os.path.basename(urlparse_local.urlparse(url)[2]) 159 localFile = os.path.join(root,'_d_'+filename) 160 ext = os.path.splitext(localFile)[1] 161 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 162 raise RuntimeError('Unknown compression type in URL: '+ url) 163 self.logPrint('Downloading '+url+' to '+localFile) 164 if os.path.exists(localFile): 165 os.unlink(localFile) 166 167 try: 168 sav_timeout = socket.getdefaulttimeout() 169 socket.setdefaulttimeout(30) 170 urlretrieve(url, localFile) 171 socket.setdefaulttimeout(sav_timeout) 172 except Exception as e: 173 socket.setdefaulttimeout(sav_timeout) 174 failureMessage = '''\ 175Unable to download package %s from: %s 176* If URL specified manually - perhaps there is a typo? 177* If your network is disconnected - please reconnect and rerun ./configure 178* Or perhaps you have a firewall blocking the download 179* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 180* or you can download the above URL manually, to /yourselectedlocation/%s 181 and use the configure option: 182 --download-%s=/yourselectedlocation/%s 183''' % (package.upper(), url, filename, package, filename) 184 raise RuntimeError(failureMessage) 185 186 self.logPrint('Extracting '+localFile) 187 if ext in ['.zip','.ZIP']: 188 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 189 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 190 dirname = os.path.normpath(output[0].strip()) 191 else: 192 failureMessage = '''\ 193Downloaded package %s from: %s is not a tarball. 194[or installed python cannot process compressed files] 195* If you are behind a firewall - please fix your proxy and rerun ./configure 196 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 197* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 198* or you can download the above URL manually, to /yourselectedlocation/%s 199 and use the configure option: 200 --download-%s=/yourselectedlocation/%s 201''' % (package.upper(), url, filename, package, filename) 202 import tarfile 203 try: 204 tf = tarfile.open(os.path.join(root, localFile)) 205 except tarfile.ReadError as e: 206 raise RuntimeError(str(e)+'\n'+failureMessage) 207 if not tf: raise RuntimeError(failureMessage) 208 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 209 firstname = tf.getnames()[0] 210 if firstname == 'pax_global_header': 211 firstmember = tf.getmembers()[1] 212 else: 213 firstmember = tf.getmembers()[0] 214 # some tarfiles list packagename/ but some list packagename/filename in the first entry 215 if firstmember.isdir(): 216 dirname = firstmember.name 217 else: 218 dirname = os.path.dirname(firstmember.name) 219 tf.extractall(root) 220 tf.close() 221 222 # fix file permissions for the untared tarballs. 223 try: 224 # check if 'dirname' is set' 225 if dirname: 226 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 227 else: 228 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 229 except RuntimeError as e: 230 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 231 os.unlink(localFile) 232 return 233 234 def ftpRetrieve(self, url, root, name,force): 235 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 236 return self.genericRetrieve(url, root, name) 237 238 def httpRetrieve(self, url, root, name,force): 239 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 240 return self.genericRetrieve(url, root, name) 241 242 def fileRetrieve(self, url, root, name,force): 243 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 244 return self.genericRetrieve(url, root, name) 245 246 def svnRetrieve(self, url, root, name,force): 247 if not hasattr(self.sourceControl, 'svn'): 248 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 249 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 250 try: 251 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name), log = self.log) 252 except RuntimeError: 253 pass 254 255 256