1from __future__ import absolute_import 2import logger 3 4import os 5import urllib 6import urlparse 7import config.base 8import socket 9 10# Fix parsing for nonstandard schemes 11urlparse.uses_netloc.extend(['bk', 'ssh', 'svn']) 12 13class Retriever(logger.Logger): 14 def __init__(self, sourceControl, clArgs = None, argDB = None): 15 logger.Logger.__init__(self, clArgs, argDB) 16 self.sourceControl = sourceControl 17 self.stamp = None 18 return 19 20 def getAuthorizedUrl(self, url): 21 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 22 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 23 if not location: 24 url = urlparse.urlunparse(('', '', path, parameters, query, fragment)) 25 authUrl = None 26 wasAuth = 0 27 else: 28 index = location.find('@') 29 if index >= 0: 30 login = location[0:index] 31 authUrl = url 32 url = urlparse.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 33 wasAuth = 1 34 else: 35 login = location.split('.')[0] 36 authUrl = urlparse.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 37 wasAuth = 0 38 return (url, authUrl, wasAuth) 39 40 def testAuthorizedUrl(self, authUrl): 41 '''Raise an exception if the URL cannot receive an SSH login without a password''' 42 if not authUrl: 43 raise RuntimeError('Url is empty') 44 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(authUrl) 45 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location, log = self.log) 46 47 def genericRetrieve(self, url, root, package): 48 '''Fetch the gzipped tarfile indicated by url and expand it into root 49 - All the logic for removing old versions, updating etc. must move''' 50 51 # copy a directory 52 if url.startswith('dir://'): 53 import shutil 54 dir = url[6:] 55 if not os.path.isdir(dir): raise RuntimeError('Url begins with dir:// but is not a directory') 56 57 if os.path.isdir(os.path.join(root,os.path.basename(dir))): shutil.rmtree(os.path.join(root,os.path.basename(dir))) 58 if os.path.isfile(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 59 60 shutil.copytree(dir,os.path.join(root,os.path.basename(dir))) 61 return 62 63 if url.startswith('git://'): 64 if not hasattr(self.sourceControl, 'git'): return 65 import shutil 66 dir = url[6:] 67 if os.path.isdir(dir): 68 if not os.path.isdir(os.path.join(dir,'.git')): raise RuntimeError('Url begins with git:// and is a directory but but does not have a .git subdirectory') 69 70 newgitrepo = os.path.join(root,'git.'+package) 71 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 72 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 73 74 try: 75 config.base.Configure.executeShellCommand(self.sourceControl.git+' clone '+dir+' '+newgitrepo, log = self.log) 76 except RuntimeError as e: 77 self.logPrint('ERROR: '+str(e)) 78 err = str(e) 79 failureMessage = '''\ 80Unable to download package %s from: %s 81* If URL specified manually - perhaps there is a typo? 82* If your network is disconnected - please reconnect and rerun ./configure 83* Or perhaps you have a firewall blocking the download 84* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 85* or you can download the above URL manually, to /yourselectedlocation 86 and use the configure option: 87 --download-%s=/yourselectedlocation 88''' % (package.upper(), url, package) 89 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 90 return 91 92 if url.startswith('hg://'): 93 if not hasattr(self.sourceControl, 'hg'): return 94 95 newgitrepo = os.path.join(root,'hg.'+package) 96 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 97 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 98 try: 99 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url[5:]+' '+newgitrepo) 100 except RuntimeError as e: 101 self.logPrint('ERROR: '+str(e)) 102 err = str(e) 103 failureMessage = '''\ 104Unable to download package %s from: %s 105* If URL specified manually - perhaps there is a typo? 106* If your network is disconnected - please reconnect and rerun ./configure 107* Or perhaps you have a firewall blocking the download 108* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 109* or you can download the above URL manually, to /yourselectedlocation 110 and use the configure option: 111 --download-%s=/yourselectedlocation 112''' % (package.upper(), url, package) 113 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 114 return 115 116 if url.startswith('ssh://hg@'): 117 if not hasattr(self.sourceControl, 'hg'): return 118 119 newgitrepo = os.path.join(root,'hg.'+package) 120 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 121 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 122 try: 123 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url+' '+newgitrepo) 124 except RuntimeError as e: 125 self.logPrint('ERROR: '+str(e)) 126 err = str(e) 127 failureMessage = '''\ 128Unable to download package %s from: %s 129* If URL specified manually - perhaps there is a typo? 130* If your network is disconnected - please reconnect and rerun ./configure 131* Or perhaps you have a firewall blocking the download 132* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 133* or you can download the above URL manually, to /yourselectedlocation 134 and use the configure option: 135 --download-%s=/yourselectedlocation 136''' % (package.upper(), url, package) 137 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 138 return 139 140 # get the tarball file name from the URL 141 filename = os.path.basename(urlparse.urlparse(url)[2]) 142 localFile = os.path.join(root,'_d_'+filename) 143 ext = os.path.splitext(localFile)[1] 144 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 145 raise RuntimeError('Unknown compression type in URL: '+ url) 146 self.logPrint('Downloading '+url+' to '+localFile) 147 if os.path.exists(localFile): 148 os.unlink(localFile) 149 150 try: 151 sav_timeout = socket.getdefaulttimeout() 152 socket.setdefaulttimeout(30) 153 urllib.urlretrieve(url, localFile) 154 socket.setdefaulttimeout(sav_timeout) 155 except Exception as e: 156 socket.setdefaulttimeout(sav_timeout) 157 failureMessage = '''\ 158Unable to download package %s from: %s 159* If URL specified manually - perhaps there is a typo? 160* If your network is disconnected - please reconnect and rerun ./configure 161* Or perhaps you have a firewall blocking the download 162* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 163* or you can download the above URL manually, to /yourselectedlocation/%s 164 and use the configure option: 165 --download-%s=/yourselectedlocation/%s 166''' % (package.upper(), url, filename, package, filename) 167 raise RuntimeError(failureMessage) 168 169 self.logPrint('Extracting '+localFile) 170 if ext in ['.zip','.ZIP']: 171 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 172 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 173 dirname = os.path.normpath(output[0].strip()) 174 else: 175 failureMessage = '''\ 176Downloaded package %s from: %s is not a tarball. 177[or installed python cannot process compressed files] 178* If you are behind a firewall - please fix your proxy and rerun ./configure 179 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 180* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 181* or you can download the above URL manually, to /yourselectedlocation/%s 182 and use the configure option: 183 --download-%s=/yourselectedlocation/%s 184''' % (package.upper(), url, filename, package, filename) 185 import tarfile 186 try: 187 tf = tarfile.open(os.path.join(root, localFile)) 188 except tarfile.ReadError as e: 189 raise RuntimeError(str(e)+'\n'+failureMessage) 190 if not tf: raise RuntimeError(failureMessage) 191 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 192 firstname = tf.getnames()[0] 193 if firstname == 'pax_global_header': 194 firstmember = tf.getmembers()[1] 195 else: 196 firstmember = tf.getmembers()[0] 197 # some tarfiles list packagename/ but some list packagename/filename in the first entry 198 if firstmember.isdir(): 199 dirname = firstmember.name 200 else: 201 dirname = os.path.dirname(firstmember.name) 202 if hasattr(tf,'extractall'): #python 2.5+ 203 tf.extractall(root) 204 else: 205 for tfile in tf.getmembers(): 206 tf.extract(tfile,root) 207 tf.close() 208 209 # fix file permissions for the untared tarballs. 210 try: 211 # check if 'dirname' is set' 212 if dirname: 213 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 214 else: 215 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 216 except RuntimeError as e: 217 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 218 os.unlink(localFile) 219 return 220 221 def ftpRetrieve(self, url, root, name,force): 222 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 223 return self.genericRetrieve(url, root, name) 224 225 def httpRetrieve(self, url, root, name,force): 226 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 227 return self.genericRetrieve(url, root, name) 228 229 def fileRetrieve(self, url, root, name,force): 230 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 231 return self.genericRetrieve(url, root, name) 232 233 def svnRetrieve(self, url, root, name,force): 234 if not hasattr(self.sourceControl, 'svn'): 235 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 236 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 237 try: 238 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name), log = self.log) 239 except RuntimeError: 240 pass 241 242 243