1import logger 2 3import os 4import urllib 5import urlparse 6import config.base 7import socket 8 9# Fix parsing for nonstandard schemes 10urlparse.uses_netloc.extend(['bk', 'ssh', 'svn']) 11 12class Retriever(logger.Logger): 13 def __init__(self, sourceControl, clArgs = None, argDB = None): 14 logger.Logger.__init__(self, clArgs, argDB) 15 self.sourceControl = sourceControl 16 self.stamp = None 17 return 18 19 def getAuthorizedUrl(self, url): 20 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 21 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 22 if not location: 23 url = urlparse.urlunparse(('', '', path, parameters, query, fragment)) 24 authUrl = None 25 wasAuth = 0 26 else: 27 index = location.find('@') 28 if index >= 0: 29 login = location[0:index] 30 authUrl = url 31 url = urlparse.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 32 wasAuth = 1 33 else: 34 login = location.split('.')[0] 35 authUrl = urlparse.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 36 wasAuth = 0 37 return (url, authUrl, wasAuth) 38 39 def testAuthorizedUrl(self, authUrl): 40 '''Raise an exception if the URL cannot receive an SSH login without a password''' 41 if not authUrl: 42 raise RuntimeError('Url is empty') 43 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(authUrl) 44 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location, log = self.log) 45 46 def genericRetrieve(self, url, root, package): 47 '''Fetch the gzipped tarfile indicated by url and expand it into root 48 - All the logic for removing old versions, updating etc. must move''' 49 50 # copy a directory 51 if url.startswith('dir://'): 52 import shutil 53 dir = url[6:] 54 if not os.path.isdir(dir): raise RuntimeError('Url begins with dir:// but is not a directory') 55 56 if os.path.isdir(os.path.join(root,os.path.basename(dir))): shutil.rmtree(os.path.join(root,os.path.basename(dir))) 57 if os.path.isfile(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 58 59 shutil.copytree(dir,os.path.join(root,os.path.basename(dir))) 60 return 61 62 if url.startswith('git://'): 63 if not hasattr(self.sourceControl, 'git'): return 64 import shutil 65 dir = url[6:] 66 if os.path.isdir(dir): 67 if not os.path.isdir(os.path.join(dir,'.git')): raise RuntimeError('Url begins with git:// and is a directory but but does not have a .git subdirectory') 68 69 newgitrepo = os.path.join(root,'git.'+package) 70 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 71 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 72 73 try: 74 config.base.Configure.executeShellCommand(self.sourceControl.git+' clone '+dir+' '+newgitrepo, log = self.log) 75 except RuntimeError, e: 76 self.logPrint('ERROR: '+str(e)) 77 err = str(e) 78 failureMessage = '''\ 79Unable to download package %s from: %s 80* If URL specified manually - perhaps there is a typo? 81* If your network is disconnected - please reconnect and rerun ./configure 82* Or perhaps you have a firewall blocking the download 83* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 84* or you can download the above URL manually, to /yourselectedlocation 85 and use the configure option: 86 --download-%s=/yourselectedlocation 87''' % (package.upper(), url, package) 88 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 89 return 90 91 if url.startswith('hg://'): 92 if not hasattr(self.sourceControl, 'hg'): return 93 94 newgitrepo = os.path.join(root,'hg.'+package) 95 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 96 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 97 try: 98 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url[5:]+' '+newgitrepo) 99 except RuntimeError, e: 100 self.logPrint('ERROR: '+str(e)) 101 err = str(e) 102 failureMessage = '''\ 103Unable to download package %s from: %s 104* If URL specified manually - perhaps there is a typo? 105* If your network is disconnected - please reconnect and rerun ./configure 106* Or perhaps you have a firewall blocking the download 107* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 108* or you can download the above URL manually, to /yourselectedlocation 109 and use the configure option: 110 --download-%s=/yourselectedlocation 111''' % (package.upper(), url, package) 112 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 113 return 114 115 if url.startswith('ssh://hg@'): 116 if not hasattr(self.sourceControl, 'hg'): return 117 118 newgitrepo = os.path.join(root,'hg.'+package) 119 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 120 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 121 try: 122 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url+' '+newgitrepo) 123 except RuntimeError, e: 124 self.logPrint('ERROR: '+str(e)) 125 err = str(e) 126 failureMessage = '''\ 127Unable to download package %s from: %s 128* If URL specified manually - perhaps there is a typo? 129* If your network is disconnected - please reconnect and rerun ./configure 130* Or perhaps you have a firewall blocking the download 131* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 132* or you can download the above URL manually, to /yourselectedlocation 133 and use the configure option: 134 --download-%s=/yourselectedlocation 135''' % (package.upper(), url, package) 136 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 137 return 138 139 # get the tarball file name from the URL 140 filename = os.path.basename(urlparse.urlparse(url)[2]) 141 localFile = os.path.join(root,'_d_'+filename) 142 ext = os.path.splitext(localFile)[1] 143 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 144 raise RuntimeError('Unknown compression type in URL: '+ url) 145 self.logPrint('Downloading '+url+' to '+localFile) 146 if os.path.exists(localFile): 147 os.unlink(localFile) 148 149 try: 150 sav_timeout = socket.getdefaulttimeout() 151 socket.setdefaulttimeout(30) 152 urllib.urlretrieve(url, localFile) 153 socket.setdefaulttimeout(sav_timeout) 154 except Exception, e: 155 socket.setdefaulttimeout(sav_timeout) 156 failureMessage = '''\ 157Unable to download package %s from: %s 158* If URL specified manually - perhaps there is a typo? 159* If your network is disconnected - please reconnect and rerun ./configure 160* Or perhaps you have a firewall blocking the download 161* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 162* or you can download the above URL manually, to /yourselectedlocation/%s 163 and use the configure option: 164 --download-%s=/yourselectedlocation/%s 165''' % (package.upper(), url, filename, package, filename) 166 raise RuntimeError(failureMessage) 167 168 self.logPrint('Extracting '+localFile) 169 if ext in ['.zip','.ZIP']: 170 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 171 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 172 dirname = os.path.normpath(output[0].strip()) 173 else: 174 failureMessage = '''\ 175Downloaded package %s from: %s is not a tarball. 176[or installed python cannot process compressed files] 177* If you are behind a firewall - please fix your proxy and rerun ./configure 178 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 179* You can run with --with-packages-dir=/adirectory and ./configure will instruct you what packages to download manually 180* or you can download the above URL manually, to /yourselectedlocation/%s 181 and use the configure option: 182 --download-%s=/yourselectedlocation/%s 183''' % (package.upper(), url, filename, package, filename) 184 import tarfile 185 try: 186 tf = tarfile.open(os.path.join(root, localFile)) 187 except tarfile.ReadError, e: 188 raise RuntimeError(str(e)+'\n'+failureMessage) 189 if not tf: raise RuntimeError(failureMessage) 190 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 191 firstname = tf.getnames()[0] 192 if firstname == 'pax_global_header': 193 firstmember = tf.getmembers()[1] 194 else: 195 firstmember = tf.getmembers()[0] 196 # some tarfiles list packagename/ but some list packagename/filename in the first entry 197 if firstmember.isdir(): 198 dirname = firstmember.name 199 else: 200 dirname = os.path.dirname(firstmember.name) 201 if hasattr(tf,'extractall'): #python 2.5+ 202 tf.extractall(root) 203 else: 204 for tfile in tf.getmembers(): 205 tf.extract(tfile,root) 206 tf.close() 207 208 # fix file permissions for the untared tarballs. 209 try: 210 # check if 'dirname' is set' 211 if dirname: 212 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 213 else: 214 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 215 except RuntimeError, e: 216 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 217 os.unlink(localFile) 218 return 219 220 def ftpRetrieve(self, url, root, name,force): 221 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 222 return self.genericRetrieve(url, root, name) 223 224 def httpRetrieve(self, url, root, name,force): 225 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 226 return self.genericRetrieve(url, root, name) 227 228 def fileRetrieve(self, url, root, name,force): 229 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 230 return self.genericRetrieve(url, root, name) 231 232 def svnRetrieve(self, url, root, name,force): 233 if not hasattr(self.sourceControl, 'svn'): 234 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 235 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 236 try: 237 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name), log = self.log) 238 except RuntimeError: 239 pass 240 241 242